{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 1.使用numpy建立神经网络并且模拟反向传播训练过程\n",
    "\"\"\"\n",
    "x 是输入，y是输出\n",
    "该模型的目的是通过反向传播修改w1,w2使得能够从x经过神经网络得到y2\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 25955205.659201466\n",
      "1 21650818.96286214\n",
      "2 19556384.384293556\n",
      "3 17403057.165490195\n",
      "4 14475387.246745165\n",
      "5 11044662.609404236\n",
      "6 7817525.762113838\n",
      "7 5262929.620382443\n",
      "8 3485381.0689438833\n",
      "9 2335114.706648213\n",
      "10 1617798.5956343613\n",
      "11 1170595.5987526367\n",
      "12 886162.6443242859\n",
      "13 698327.3130974998\n",
      "14 568518.4456842763\n",
      "15 474562.3255670698\n",
      "16 403472.53933160054\n",
      "17 347673.06295218866\n",
      "18 302574.69317528734\n",
      "19 265303.6255274894\n",
      "20 234021.73311427655\n",
      "21 207452.8917569713\n",
      "22 184622.77207224304\n",
      "23 164855.23864557542\n",
      "24 147615.01585902434\n",
      "25 132515.1510898369\n",
      "26 119230.83456569105\n",
      "27 107493.93970871824\n",
      "28 97115.90597064739\n",
      "29 87898.2071276091\n",
      "30 79688.0195557669\n",
      "31 72370.79749850361\n",
      "32 65821.46233202779\n",
      "33 59950.22284054712\n",
      "34 54673.287782910105\n",
      "35 49920.11529123422\n",
      "36 45634.42312704044\n",
      "37 41764.79741394466\n",
      "38 38263.04534264824\n",
      "39 35089.515043998654\n",
      "40 32209.32969099384\n",
      "41 29592.42693990157\n",
      "42 27211.26512938907\n",
      "43 25042.497828642197\n",
      "44 23065.604752249856\n",
      "45 21262.035407448275\n",
      "46 19615.299195805706\n",
      "47 18111.44107154256\n",
      "48 16735.667885389783\n",
      "49 15476.68908566606\n",
      "50 14321.917300032394\n",
      "51 13261.493339784869\n",
      "52 12286.938607096397\n",
      "53 11390.73009141329\n",
      "54 10565.90568849472\n",
      "55 9806.574250575337\n",
      "56 9106.788458311432\n",
      "57 8461.249576074135\n",
      "58 7865.368918411466\n",
      "59 7315.033782197126\n",
      "60 6806.64777683345\n",
      "61 6336.550248440305\n",
      "62 5901.606960743759\n",
      "63 5498.834917272729\n",
      "64 5125.841223544449\n",
      "65 4780.172739176423\n",
      "66 4459.694922388728\n",
      "67 4162.313592666224\n",
      "68 3886.3674649219233\n",
      "69 3630.060559312923\n",
      "70 3391.9408013932234\n",
      "71 3170.681446418987\n",
      "72 2964.8986136135595\n",
      "73 2773.424695050754\n",
      "74 2595.2403322430428\n",
      "75 2429.401207678319\n",
      "76 2274.880164779129\n",
      "77 2130.8779424536374\n",
      "78 1996.606066916729\n",
      "79 1871.400822803338\n",
      "80 1754.6767377362312\n",
      "81 1645.8045825379188\n",
      "82 1544.1629782382372\n",
      "83 1449.2102003914974\n",
      "84 1360.5029568350826\n",
      "85 1277.6437484049843\n",
      "86 1200.162150701275\n",
      "87 1127.723528085724\n",
      "88 1059.9365939302952\n",
      "89 996.4802405027102\n",
      "90 937.048025584797\n",
      "91 881.3950265637119\n",
      "92 829.2486811911607\n",
      "93 780.3634481044437\n",
      "94 734.554119135862\n",
      "95 691.5723543027232\n",
      "96 651.2720377809198\n",
      "97 613.4435688646532\n",
      "98 577.9366632087524\n",
      "99 544.6111631787253\n",
      "100 513.331829941026\n",
      "101 483.92678667820326\n",
      "102 456.2968330707248\n",
      "103 430.33163004993224\n",
      "104 405.9280757149879\n",
      "105 382.98918859657664\n",
      "106 361.4071216883609\n",
      "107 341.1082203107313\n",
      "108 322.004310182247\n",
      "109 304.03768517510196\n",
      "110 287.116909413282\n",
      "111 271.1856528958894\n",
      "112 256.1768619862886\n",
      "113 242.04800236390912\n",
      "114 228.72881521505317\n",
      "115 216.175030049915\n",
      "116 204.3442716458771\n",
      "117 193.20289554190558\n",
      "118 182.68988595667622\n",
      "119 172.7700796721678\n",
      "120 163.41191831052913\n",
      "121 154.5870867736578\n",
      "122 146.2579258883053\n",
      "123 138.3975360830753\n",
      "124 130.97600093358767\n",
      "125 123.97301208402135\n",
      "126 117.36026086362159\n",
      "127 111.11042158411004\n",
      "128 105.20664505087554\n",
      "129 99.63226901831766\n",
      "130 94.3639977191446\n",
      "131 89.38231372929678\n",
      "132 84.67349945777681\n",
      "133 80.22339622622805\n",
      "134 76.01793198827446\n",
      "135 72.04004566953172\n",
      "136 68.27611092530478\n",
      "137 64.71575654113495\n",
      "138 61.34924953652994\n",
      "139 58.1618283140203\n",
      "140 55.14609049396782\n",
      "141 52.29323380473562\n",
      "142 49.59266578567088\n",
      "143 47.0343257804172\n",
      "144 44.61201657781077\n",
      "145 42.31896874227533\n",
      "146 40.148454806372044\n",
      "147 38.092220345790196\n",
      "148 36.14389467973399\n",
      "149 34.297645990285716\n",
      "150 32.54935047657347\n",
      "151 30.891827750628607\n",
      "152 29.321310765750745\n",
      "153 27.83314093613089\n",
      "154 26.423251428560278\n",
      "155 25.085547557376138\n",
      "156 23.817336434052233\n",
      "157 22.614935207116226\n",
      "158 21.475592172271522\n",
      "159 20.394778915249137\n",
      "160 19.3695458818063\n",
      "161 18.39691913459685\n",
      "162 17.474698708953582\n",
      "163 16.59944360291412\n",
      "164 15.769107880073168\n",
      "165 14.981350339558148\n",
      "166 14.234283531214377\n",
      "167 13.524903495279133\n",
      "168 12.851468657660448\n",
      "169 12.212283270804885\n",
      "170 11.606046904570494\n",
      "171 11.03026153206719\n",
      "172 10.483590081885913\n",
      "173 9.964480556304938\n",
      "174 9.471806788378998\n",
      "175 9.003816271210109\n",
      "176 8.559481199973273\n",
      "177 8.137439354966041\n",
      "178 7.736753637193267\n",
      "179 7.35598500778622\n",
      "180 6.994289879341053\n",
      "181 6.650727744936393\n",
      "182 6.3245845721820135\n",
      "183 6.014529968381346\n",
      "184 5.719859349023803\n",
      "185 5.439878636533765\n",
      "186 5.1739688277053295\n",
      "187 4.921263551507957\n",
      "188 4.681066006655165\n",
      "189 4.452713896518811\n",
      "190 4.235789544314034\n",
      "191 4.029505620150751\n",
      "192 3.833461990418704\n",
      "193 3.6471516142577602\n",
      "194 3.4700393363985085\n",
      "195 3.301561193889625\n",
      "196 3.141383874370108\n",
      "197 2.989097771074511\n",
      "198 2.844445984604212\n",
      "199 2.7067625630054617\n",
      "200 2.5758198385467774\n",
      "201 2.4512924551983923\n",
      "202 2.3329411584512036\n",
      "203 2.2203543689467544\n",
      "204 2.1132631018447836\n",
      "205 2.0113752321907024\n",
      "206 1.9145211762518897\n",
      "207 1.822345817526867\n",
      "208 1.7346672286712699\n",
      "209 1.651270431610667\n",
      "210 1.5719543881297273\n",
      "211 1.4964485249195596\n",
      "212 1.4246262169339419\n",
      "213 1.35628039672659\n",
      "214 1.291296134786244\n",
      "215 1.2294256170245974\n",
      "216 1.170546755816331\n",
      "217 1.1145279183775552\n",
      "218 1.0612333578954134\n",
      "219 1.010493038600476\n",
      "220 0.9622121012618228\n",
      "221 0.9162658219079636\n",
      "222 0.87255167881922\n",
      "223 0.8309160411858125\n",
      "224 0.7912874032806868\n",
      "225 0.7535865630777385\n",
      "226 0.7177120414585281\n",
      "227 0.6835319095589607\n",
      "228 0.6509934793262805\n",
      "229 0.620019366070287\n",
      "230 0.5905655039899202\n",
      "231 0.5624946937645945\n",
      "232 0.535766960839885\n",
      "233 0.510321531669263\n",
      "234 0.48611623133856063\n",
      "235 0.46305645098006315\n",
      "236 0.44109539252852714\n",
      "237 0.42018327265399047\n",
      "238 0.4002856946566893\n",
      "239 0.38132566627797865\n",
      "240 0.3632713676941303\n",
      "241 0.3460797613611186\n",
      "242 0.3297164960353731\n",
      "243 0.314124472608942\n",
      "244 0.29927354092116637\n",
      "245 0.2851343451032653\n",
      "246 0.2716754632926793\n",
      "247 0.2588483175907672\n",
      "248 0.24662981317892785\n",
      "249 0.23499394437025078\n",
      "250 0.2239164550075989\n",
      "251 0.21336164422831969\n",
      "252 0.203304713156232\n",
      "253 0.1937262945474408\n",
      "254 0.1846066639354026\n",
      "255 0.1759166045289171\n",
      "256 0.1676372214069196\n",
      "257 0.15975055981999758\n",
      "258 0.1522399690484566\n",
      "259 0.14508227519861877\n",
      "260 0.13826259820439654\n",
      "261 0.131767636128184\n",
      "262 0.1255800988662028\n",
      "263 0.11968320289511036\n",
      "264 0.11406389757713462\n",
      "265 0.10871257603629825\n",
      "266 0.10361442411192029\n",
      "267 0.0987545728661675\n",
      "268 0.09412352741192473\n",
      "269 0.08971318158306515\n",
      "270 0.08551091909098654\n",
      "271 0.08150492700748711\n",
      "272 0.07768735625793274\n",
      "273 0.07405159121080351\n",
      "274 0.07058580347056512\n",
      "275 0.06728293936188465\n",
      "276 0.06413547676451811\n",
      "277 0.06113712615158069\n",
      "278 0.0582786661467463\n",
      "279 0.05555420150429562\n",
      "280 0.052958713191867825\n",
      "281 0.05048564487318832\n",
      "282 0.04812766541170717\n",
      "283 0.0458798753003031\n",
      "284 0.04373835301999858\n",
      "285 0.0416982464623575\n",
      "286 0.03975230410306155\n",
      "287 0.03789761007937338\n",
      "288 0.0361308678855175\n",
      "289 0.03444652161230648\n",
      "290 0.03284083198553683\n",
      "291 0.03131034668986288\n",
      "292 0.029852297027160836\n",
      "293 0.028461672814905737\n",
      "294 0.02713615856743203\n",
      "295 0.025872817712884747\n",
      "296 0.024668993788067303\n",
      "297 0.023520698240817872\n",
      "298 0.022426077273175374\n",
      "299 0.02138320786304624\n",
      "300 0.020389008396585653\n",
      "301 0.01944077430887426\n",
      "302 0.01853680063582124\n",
      "303 0.017675703269197117\n",
      "304 0.016854404641087124\n",
      "305 0.016071201661119562\n",
      "306 0.01532473893973298\n",
      "307 0.014613334576160448\n",
      "308 0.013934688802546943\n",
      "309 0.013287820194085696\n",
      "310 0.012671265525066956\n",
      "311 0.012083300268948265\n",
      "312 0.011522605478150122\n",
      "313 0.01098813569395653\n",
      "314 0.010478759313546757\n",
      "315 0.009992885539453605\n",
      "316 0.009529532128209284\n",
      "317 0.009087888099794815\n",
      "318 0.008666892676512642\n",
      "319 0.008265267524209223\n",
      "320 0.007882377261997264\n",
      "321 0.007517334235103652\n",
      "322 0.0071692370537498745\n",
      "323 0.006837309848515672\n",
      "324 0.006520796414068874\n",
      "325 0.006219050320684243\n",
      "326 0.005931224806725736\n",
      "327 0.005656835300187402\n",
      "328 0.0053951645716310715\n",
      "329 0.005145661867982313\n",
      "330 0.004907680593231193\n",
      "331 0.004680741404556189\n",
      "332 0.00446442110390724\n",
      "333 0.004258090326569355\n",
      "334 0.004061304228370238\n",
      "335 0.0038736212831002583\n",
      "336 0.0036947038233948355\n",
      "337 0.0035240756427045213\n",
      "338 0.003361293670591997\n",
      "339 0.0032060877730797415\n",
      "340 0.003058086851068522\n",
      "341 0.0029169081054084642\n",
      "342 0.0027822650646108765\n",
      "343 0.0026539049688559255\n",
      "344 0.0025314391994196695\n",
      "345 0.0024146262037026624\n",
      "346 0.0023032707023176472\n",
      "347 0.0021970688125892077\n",
      "348 0.002095727149566692\n",
      "349 0.001999084778982222\n",
      "350 0.001906963632593194\n",
      "351 0.001819059803597923\n",
      "352 0.0017352032622563484\n",
      "353 0.0016552574918965984\n",
      "354 0.0015789966577666082\n",
      "355 0.0015062421984198757\n",
      "356 0.00143686144786103\n",
      "357 0.001370707953143918\n",
      "358 0.00130757274591546\n",
      "359 0.0012473566217192234\n",
      "360 0.001189963683678956\n",
      "361 0.001135183629698503\n",
      "362 0.001082922255109908\n",
      "363 0.0010330953575856474\n",
      "364 0.0009855678935249915\n",
      "365 0.0009402197526161076\n",
      "366 0.0008969665649382775\n",
      "367 0.0008557198503484633\n",
      "368 0.0008163584604318652\n",
      "369 0.0007788189954642546\n",
      "370 0.0007430182993857646\n",
      "371 0.000708861274312685\n",
      "372 0.0006762735385134979\n",
      "373 0.000645192240637072\n",
      "374 0.0006155515472047282\n",
      "375 0.0005872609597819135\n",
      "376 0.0005602807677959132\n",
      "377 0.0005345460741460551\n",
      "378 0.0005099902026197275\n",
      "379 0.00048657191786419826\n",
      "380 0.00046422755258749773\n",
      "381 0.0004429118413716357\n",
      "382 0.00042257651902850065\n",
      "383 0.0004031790856430504\n",
      "384 0.00038467616028024066\n",
      "385 0.0003670205494626483\n",
      "386 0.0003501749700114005\n",
      "387 0.0003341111215784156\n",
      "388 0.00031878349992596656\n",
      "389 0.00030415686320403796\n",
      "390 0.0002902066563338564\n",
      "391 0.00027689679885120133\n",
      "392 0.0002641982291614532\n",
      "393 0.00025208385079780905\n",
      "394 0.00024052776051153844\n",
      "395 0.00022949978400779521\n",
      "396 0.00021897782152602798\n",
      "397 0.00020894536236899916\n",
      "398 0.00019936854562041523\n",
      "399 0.00019023031896127167\n",
      "400 0.00018151635744662712\n",
      "401 0.00017320013027069653\n",
      "402 0.00016526456091399343\n",
      "403 0.00015769582544111014\n",
      "404 0.0001504734754775841\n",
      "405 0.00014358102040854063\n",
      "406 0.0001370072473960489\n",
      "407 0.0001307352458081064\n",
      "408 0.00012474859048874006\n",
      "409 0.00011903780903405555\n",
      "410 0.00011359065814754588\n",
      "411 0.0001083911774294217\n",
      "412 0.00010343023607024742\n",
      "413 9.86987214570167e-05\n",
      "414 9.418156981113928e-05\n",
      "415 8.987233230929993e-05\n",
      "416 8.576304696709075e-05\n",
      "417 8.183913773588852e-05\n",
      "418 7.809520410402363e-05\n",
      "419 7.452535704057208e-05\n",
      "420 7.111684564047561e-05\n",
      "421 6.786448182357436e-05\n",
      "422 6.476293953096529e-05\n",
      "423 6.180174862875388e-05\n",
      "424 5.897603175320315e-05\n",
      "425 5.628187794237327e-05\n",
      "426 5.3709083459639275e-05\n",
      "427 5.1254057708748297e-05\n",
      "428 4.891306005227474e-05\n",
      "429 4.667789666127154e-05\n",
      "430 4.454506670768098e-05\n",
      "431 4.2510851606993296e-05\n",
      "432 4.056865604458969e-05\n",
      "433 3.871520629966455e-05\n",
      "434 3.6947931979740015e-05\n",
      "435 3.526043486420014e-05\n",
      "436 3.3650027893885164e-05\n",
      "437 3.2114176529368574e-05\n",
      "438 3.064778928521999e-05\n",
      "439 2.924857361903445e-05\n",
      "440 2.791403167702228e-05\n",
      "441 2.6639717745344357e-05\n",
      "442 2.5423578438370702e-05\n",
      "443 2.4263899713247563e-05\n",
      "444 2.3156572665558833e-05\n",
      "445 2.209980015791736e-05\n",
      "446 2.1091941706193123e-05\n",
      "447 2.01295252111199e-05\n",
      "448 1.9211154717795585e-05\n",
      "449 1.833542376344345e-05\n",
      "450 1.7498970080910607e-05\n",
      "451 1.670079436820924e-05\n",
      "452 1.5939517094702544e-05\n",
      "453 1.521259880841004e-05\n",
      "454 1.4518972347836073e-05\n",
      "455 1.3857267805792221e-05\n",
      "456 1.3225435619025611e-05\n",
      "457 1.2622493169200155e-05\n",
      "458 1.2047412743582877e-05\n",
      "459 1.1498253449022648e-05\n",
      "460 1.0974273212599554e-05\n",
      "461 1.0474350239941824e-05\n",
      "462 9.99697764871388e-06\n",
      "463 9.541551033307066e-06\n",
      "464 9.106973582759068e-06\n",
      "465 8.69202399785942e-06\n",
      "466 8.296122572928034e-06\n",
      "467 7.91830498543275e-06\n",
      "468 7.557648695962957e-06\n",
      "469 7.21355366241176e-06\n",
      "470 6.885095955092931e-06\n",
      "471 6.571540280316796e-06\n",
      "472 6.272422919625957e-06\n",
      "473 5.986891406500391e-06\n",
      "474 5.714304724274737e-06\n",
      "475 5.454247969195921e-06\n",
      "476 5.205976343046834e-06\n",
      "477 4.969021645203138e-06\n",
      "478 4.742951453782431e-06\n",
      "479 4.527099222033496e-06\n",
      "480 4.321105418275534e-06\n",
      "481 4.1245103837500825e-06\n",
      "482 3.936850238285502e-06\n",
      "483 3.75779839597974e-06\n",
      "484 3.5868622185676335e-06\n",
      "485 3.4236955086992563e-06\n",
      "486 3.268003081006096e-06\n",
      "487 3.119369482982629e-06\n",
      "488 2.9775353409062543e-06\n",
      "489 2.8421368793724725e-06\n",
      "490 2.71290779331091e-06\n",
      "491 2.5895652516333435e-06\n",
      "492 2.471826026594531e-06\n",
      "493 2.3594802642779933e-06\n",
      "494 2.252242681447833e-06\n",
      "495 2.149849790034356e-06\n",
      "496 2.0521525237039016e-06\n",
      "497 1.958890705631034e-06\n",
      "498 1.8698648967829164e-06\n",
      "499 1.7849162976690959e-06\n"
     ]
    }
   ],
   "source": [
    "# 可运行代码见本文件夹中的 two_layer_net_numpy.py\n",
    "import numpy as np\n",
    "\n",
    "# N是批大小；D_in是输入维度\n",
    "# H是隐藏层维度；D_out是输出维度\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 产生随机输入和输出数据\n",
    "x = np.random.randn(N, D_in)\n",
    "y = np.random.randn(N, D_out)\n",
    "\n",
    "# 随机初始化权重\n",
    "w1 = np.random.randn(D_in, H)\n",
    "w2 = np.random.randn(H, D_out)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # 前向传播：计算预测值y\n",
    "    # 用的矩阵乘法（点乘）\n",
    "    h = x.dot(w1)\n",
    "    # 矩阵中的每一个值和0比，取大的，去掉负值（模拟relu激活函数）\n",
    "    h_relu = np.maximum(h, 0)\n",
    "    y_pred = h_relu.dot(w2)\n",
    "\n",
    "    # 计算并显示loss(损失）\n",
    "    loss = np.square(y_pred - y).sum()\n",
    "    print(t, loss)\n",
    "\n",
    "    # 反向传播，计算w1、w2对loss的梯度\n",
    "    grad_y_pred = 2.0 * (y_pred - y)\n",
    "    grad_w2 = h_relu.T.dot(grad_y_pred)\n",
    "    grad_h_relu = grad_y_pred.dot(w2.T)\n",
    "    grad_h = grad_h_relu.copy()\n",
    "    grad_h[h < 0] = 0\n",
    "    grad_w1 = x.T.dot(grad_h)\n",
    "\n",
    "    # 更新权重\n",
    "    w1 -= learning_rate * grad_w1\n",
    "    w2 -= learning_rate * grad_w2"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 33529722.0\n",
      "1 26066816.0\n",
      "2 20151620.0\n",
      "3 14525750.0\n",
      "4 9772366.0\n",
      "5 6320982.0\n",
      "6 4101297.0\n",
      "7 2753514.5\n",
      "8 1945894.75\n",
      "9 1449287.375\n",
      "10 1129003.75\n",
      "11 910183.375\n",
      "12 751842.0\n",
      "13 631836.75\n",
      "14 537484.3125\n",
      "15 461381.0\n",
      "16 398803.9375\n",
      "17 346610.4375\n",
      "18 302657.8125\n",
      "19 265351.0625\n",
      "20 233556.484375\n",
      "21 206320.5625\n",
      "22 182836.84375\n",
      "23 162496.4375\n",
      "24 144783.0\n",
      "25 129316.984375\n",
      "26 115750.5546875\n",
      "27 103818.796875\n",
      "28 93295.296875\n",
      "29 83984.546875\n",
      "30 75730.2109375\n",
      "31 68392.453125\n",
      "32 61855.77734375\n",
      "33 56023.73828125\n",
      "34 50809.921875\n",
      "35 46141.82421875\n",
      "36 41957.9140625\n",
      "37 38198.25390625\n",
      "38 34815.63671875\n",
      "39 31765.91796875\n",
      "40 29009.640625\n",
      "41 26518.46875\n",
      "42 24264.1875\n",
      "43 22222.716796875\n",
      "44 20370.900390625\n",
      "45 18688.87890625\n",
      "46 17159.62890625\n",
      "47 15767.1669921875\n",
      "48 14498.546875\n",
      "49 13341.7861328125\n",
      "50 12285.73828125\n",
      "51 11322.3369140625\n",
      "52 10442.302734375\n",
      "53 9636.9814453125\n",
      "54 8898.6162109375\n",
      "55 8221.3837890625\n",
      "56 7600.1591796875\n",
      "57 7030.697265625\n",
      "58 6507.9482421875\n",
      "59 6027.40869140625\n",
      "60 5585.37890625\n",
      "61 5177.9052734375\n",
      "62 4802.31494140625\n",
      "63 4455.95654296875\n",
      "64 4136.39306640625\n",
      "65 3841.32470703125\n",
      "66 3568.6630859375\n",
      "67 3316.700439453125\n",
      "68 3083.61767578125\n",
      "69 2867.99951171875\n",
      "70 2668.4208984375\n",
      "71 2483.5927734375\n",
      "72 2312.26318359375\n",
      "73 2153.464599609375\n",
      "74 2006.2467041015625\n",
      "75 1869.6656494140625\n",
      "76 1742.913330078125\n",
      "77 1625.20947265625\n",
      "78 1515.8555908203125\n",
      "79 1414.2774658203125\n",
      "80 1319.87353515625\n",
      "81 1232.140625\n",
      "82 1150.5753173828125\n",
      "83 1074.6513671875\n",
      "84 1004.0067749023438\n",
      "85 938.2298583984375\n",
      "86 876.9528198242188\n",
      "87 819.8509521484375\n",
      "88 766.6555786132812\n",
      "89 717.06787109375\n",
      "90 670.8326416015625\n",
      "91 627.7062377929688\n",
      "92 587.4874877929688\n",
      "93 550.088134765625\n",
      "94 515.173583984375\n",
      "95 482.5777893066406\n",
      "96 452.119384765625\n",
      "97 423.6659240722656\n",
      "98 397.0766296386719\n",
      "99 372.24078369140625\n",
      "100 349.0252685546875\n",
      "101 327.3435974121094\n",
      "102 307.0668640136719\n",
      "103 288.09637451171875\n",
      "104 270.3429870605469\n",
      "105 253.7216796875\n",
      "106 238.16127014160156\n",
      "107 223.59915161132812\n",
      "108 209.95687866210938\n",
      "109 197.17672729492188\n",
      "110 185.20877075195312\n",
      "111 173.9893798828125\n",
      "112 163.47276306152344\n",
      "113 153.61166381835938\n",
      "114 144.36790466308594\n",
      "115 135.7024688720703\n",
      "116 127.5720443725586\n",
      "117 119.94578552246094\n",
      "118 112.79243469238281\n",
      "119 106.08098602294922\n",
      "120 99.77735137939453\n",
      "121 93.86468505859375\n",
      "122 88.31111145019531\n",
      "123 83.09848022460938\n",
      "124 78.20259094238281\n",
      "125 73.6061782836914\n",
      "126 69.28482818603516\n",
      "127 65.22811889648438\n",
      "128 61.41347885131836\n",
      "129 57.83097839355469\n",
      "130 54.4624137878418\n",
      "131 51.296875\n",
      "132 48.32229232788086\n",
      "133 45.52338409423828\n",
      "134 42.89155578613281\n",
      "135 40.415550231933594\n",
      "136 38.08868408203125\n",
      "137 35.89742660522461\n",
      "138 33.836360931396484\n",
      "139 31.89849090576172\n",
      "140 30.072906494140625\n",
      "141 28.354459762573242\n",
      "142 26.738330841064453\n",
      "143 25.215593338012695\n",
      "144 23.782562255859375\n",
      "145 22.435073852539062\n",
      "146 21.163284301757812\n",
      "147 19.9666805267334\n",
      "148 18.839031219482422\n",
      "149 17.776851654052734\n",
      "150 16.77621841430664\n",
      "151 15.8333740234375\n",
      "152 14.944832801818848\n",
      "153 14.106395721435547\n",
      "154 13.31767749786377\n",
      "155 12.573945999145508\n",
      "156 11.87183952331543\n",
      "157 11.21138858795166\n",
      "158 10.587122917175293\n",
      "159 9.999433517456055\n",
      "160 9.444847106933594\n",
      "161 8.922443389892578\n",
      "162 8.428050994873047\n",
      "163 7.962121486663818\n",
      "164 7.52357292175293\n",
      "165 7.109117031097412\n",
      "166 6.718088626861572\n",
      "167 6.349383354187012\n",
      "168 6.000916481018066\n",
      "169 5.672069549560547\n",
      "170 5.361442565917969\n",
      "171 5.0688066482543945\n",
      "172 4.7921929359436035\n",
      "173 4.531147480010986\n",
      "174 4.2845306396484375\n",
      "175 4.051745414733887\n",
      "176 3.8316049575805664\n",
      "177 3.623481273651123\n",
      "178 3.4271092414855957\n",
      "179 3.242072343826294\n",
      "180 3.067049503326416\n",
      "181 2.901806592941284\n",
      "182 2.7449002265930176\n",
      "183 2.5974576473236084\n",
      "184 2.4574220180511475\n",
      "185 2.325401544570923\n",
      "186 2.20053768157959\n",
      "187 2.082756519317627\n",
      "188 1.9714908599853516\n",
      "189 1.8657647371292114\n",
      "190 1.7664533853530884\n",
      "191 1.671964406967163\n",
      "192 1.583041787147522\n",
      "193 1.498877763748169\n",
      "194 1.4192588329315186\n",
      "195 1.343761682510376\n",
      "196 1.2726047039031982\n",
      "197 1.2052117586135864\n",
      "198 1.1412321329116821\n",
      "199 1.0807141065597534\n",
      "200 1.023674726486206\n",
      "201 0.9696059823036194\n",
      "202 0.9185341596603394\n",
      "203 0.8702815771102905\n",
      "204 0.8243277668952942\n",
      "205 0.7810927629470825\n",
      "206 0.7400468587875366\n",
      "207 0.7012709975242615\n",
      "208 0.664483904838562\n",
      "209 0.6296567320823669\n",
      "210 0.5968403816223145\n",
      "211 0.5655995011329651\n",
      "212 0.5361055731773376\n",
      "213 0.5082023739814758\n",
      "214 0.4817310571670532\n",
      "215 0.4567708969116211\n",
      "216 0.4328482449054718\n",
      "217 0.41032665967941284\n",
      "218 0.3890005648136139\n",
      "219 0.36888647079467773\n",
      "220 0.34973201155662537\n",
      "221 0.33154335618019104\n",
      "222 0.3144615888595581\n",
      "223 0.29821401834487915\n",
      "224 0.2828477621078491\n",
      "225 0.2682762145996094\n",
      "226 0.2544597089290619\n",
      "227 0.24137453734874725\n",
      "228 0.22885939478874207\n",
      "229 0.2170955091714859\n",
      "230 0.20590737462043762\n",
      "231 0.1953556388616562\n",
      "232 0.18534311652183533\n",
      "233 0.17582406103610992\n",
      "234 0.16680081188678741\n",
      "235 0.15829947590827942\n",
      "236 0.15019385516643524\n",
      "237 0.142503559589386\n",
      "238 0.1353163868188858\n",
      "239 0.12838734686374664\n",
      "240 0.12186543643474579\n",
      "241 0.11562729626893997\n",
      "242 0.10977138578891754\n",
      "243 0.10410552471876144\n",
      "244 0.09887450188398361\n",
      "245 0.09389327466487885\n",
      "246 0.08916017413139343\n",
      "247 0.08462891727685928\n",
      "248 0.08032844960689545\n",
      "249 0.07628295570611954\n",
      "250 0.07240607589483261\n",
      "251 0.06874488294124603\n",
      "252 0.06526204198598862\n",
      "253 0.0619654580950737\n",
      "254 0.05885493755340576\n",
      "255 0.05586919188499451\n",
      "256 0.05305612459778786\n",
      "257 0.050392035394907\n",
      "258 0.047875478863716125\n",
      "259 0.04548870027065277\n",
      "260 0.04317917674779892\n",
      "261 0.04099990427494049\n",
      "262 0.03894852474331856\n",
      "263 0.03698691725730896\n",
      "264 0.03514190763235092\n",
      "265 0.03339420258998871\n",
      "266 0.031728245317935944\n",
      "267 0.030146151781082153\n",
      "268 0.028644055128097534\n",
      "269 0.027211450040340424\n",
      "270 0.02584579400718212\n",
      "271 0.02457611635327339\n",
      "272 0.023361459374427795\n",
      "273 0.022215180099010468\n",
      "274 0.021107308566570282\n",
      "275 0.02008761651813984\n",
      "276 0.019092785194516182\n",
      "277 0.01814746856689453\n",
      "278 0.017246654257178307\n",
      "279 0.016395462676882744\n",
      "280 0.015595921315252781\n",
      "281 0.014840359799563885\n",
      "282 0.014107505790889263\n",
      "283 0.013429857790470123\n",
      "284 0.012755845673382282\n",
      "285 0.012146820314228535\n",
      "286 0.01154958363622427\n",
      "287 0.010974363423883915\n",
      "288 0.010436450131237507\n",
      "289 0.009940571151673794\n",
      "290 0.009453115053474903\n",
      "291 0.008993996307253838\n",
      "292 0.008571102283895016\n",
      "293 0.008165313862264156\n",
      "294 0.007774678524583578\n",
      "295 0.007402689661830664\n",
      "296 0.007056404836475849\n",
      "297 0.006722867488861084\n",
      "298 0.006404053419828415\n",
      "299 0.006106656044721603\n",
      "300 0.005822344683110714\n",
      "301 0.005553454626351595\n",
      "302 0.0052956994622945786\n",
      "303 0.005049746483564377\n",
      "304 0.004815015476197004\n",
      "305 0.0045944382436573505\n",
      "306 0.004382852464914322\n",
      "307 0.004185766912996769\n",
      "308 0.004000113811343908\n",
      "309 0.003816103097051382\n",
      "310 0.003642313415184617\n",
      "311 0.0034762343857437372\n",
      "312 0.0033223070204257965\n",
      "313 0.003174484707415104\n",
      "314 0.003034110413864255\n",
      "315 0.002898653969168663\n",
      "316 0.0027730586007237434\n",
      "317 0.0026529263705015182\n",
      "318 0.0025396854616701603\n",
      "319 0.00243176962248981\n",
      "320 0.002327741589397192\n",
      "321 0.00223028683103621\n",
      "322 0.0021360900718718767\n",
      "323 0.0020461436361074448\n",
      "324 0.001962183276191354\n",
      "325 0.0018777805380523205\n",
      "326 0.001799595425836742\n",
      "327 0.0017267437651753426\n",
      "328 0.0016558704664930701\n",
      "329 0.0015872383955866098\n",
      "330 0.0015236028702929616\n",
      "331 0.0014605571050196886\n",
      "332 0.001403300790116191\n",
      "333 0.001347204903140664\n",
      "334 0.0012944425689056516\n",
      "335 0.0012437360128387809\n",
      "336 0.0011967706959694624\n",
      "337 0.0011531213531270623\n",
      "338 0.0011096481466665864\n",
      "339 0.001068781246431172\n",
      "340 0.0010285439202561975\n",
      "341 0.0009900755248963833\n",
      "342 0.0009540952160023153\n",
      "343 0.0009183281217701733\n",
      "344 0.0008848461438901722\n",
      "345 0.0008521765121258795\n",
      "346 0.0008222354808822274\n",
      "347 0.0007935617468319833\n",
      "348 0.0007664912263862789\n",
      "349 0.0007387665682472289\n",
      "350 0.0007133293547667563\n",
      "351 0.0006875910330563784\n",
      "352 0.000664457562379539\n",
      "353 0.0006419902783818543\n",
      "354 0.0006209732382558286\n",
      "355 0.0006009499193169177\n",
      "356 0.0005821947124786675\n",
      "357 0.0005632136017084122\n",
      "358 0.0005448852898553014\n",
      "359 0.0005287906969897449\n",
      "360 0.0005120834102854133\n",
      "361 0.0004947310662828386\n",
      "362 0.000478624424431473\n",
      "363 0.000464756740257144\n",
      "364 0.00045130099169909954\n",
      "365 0.00043784346780739725\n",
      "366 0.00042419807869009674\n",
      "367 0.0004120962112210691\n",
      "368 0.00039957615081220865\n",
      "369 0.0003878588031511754\n",
      "370 0.00037728025927208364\n",
      "371 0.00036515924148261547\n",
      "372 0.0003547698725014925\n",
      "373 0.0003453263489063829\n",
      "374 0.0003353010106366128\n",
      "375 0.000325795728713274\n",
      "376 0.00031689644674770534\n",
      "377 0.00030800950480625033\n",
      "378 0.0002999443677254021\n",
      "379 0.0002917624660767615\n",
      "380 0.00028386589838191867\n",
      "381 0.00027622151537798345\n",
      "382 0.00026871453155763447\n",
      "383 0.00026187958428636193\n",
      "384 0.000254827958997339\n",
      "385 0.0002477109956089407\n",
      "386 0.00024136918364092708\n",
      "387 0.00023506057914346457\n",
      "388 0.00022841966710984707\n",
      "389 0.0002227716613560915\n",
      "390 0.00021748738072346896\n",
      "391 0.0002115736569976434\n",
      "392 0.00020645490440074354\n",
      "393 0.0002013685298152268\n",
      "394 0.00019652624905575067\n",
      "395 0.00019171032181475312\n",
      "396 0.00018738499784376472\n",
      "397 0.00018322865071240813\n",
      "398 0.00017823593225330114\n",
      "399 0.00017431212472729385\n",
      "400 0.00017021092935465276\n",
      "401 0.00016619452799204737\n",
      "402 0.0001629124890314415\n",
      "403 0.00015924495528452098\n",
      "404 0.0001555366616230458\n",
      "405 0.0001521327212685719\n",
      "406 0.00014834938338026404\n",
      "407 0.00014516834926325828\n",
      "408 0.00014239357551559806\n",
      "409 0.0001390046236338094\n",
      "410 0.0001357177534373477\n",
      "411 0.00013277347898110747\n",
      "412 0.0001301924785366282\n",
      "413 0.00012750683526974171\n",
      "414 0.00012453632371034473\n",
      "415 0.0001218438264913857\n",
      "416 0.00011949487816309556\n",
      "417 0.00011676611029542983\n",
      "418 0.00011487857409520075\n",
      "419 0.00011244992492720485\n",
      "420 0.00011075695510953665\n",
      "421 0.00010803751501953229\n",
      "422 0.00010563170508248731\n",
      "423 0.00010379034210927784\n",
      "424 0.00010173016926273704\n",
      "425 9.984878124669194e-05\n",
      "426 9.800072439247742e-05\n",
      "427 9.60955730988644e-05\n",
      "428 9.417979163117707e-05\n",
      "429 9.259499347535893e-05\n",
      "430 9.05935448827222e-05\n",
      "431 8.911010081646964e-05\n",
      "432 8.739436452742666e-05\n",
      "433 8.573923696530983e-05\n",
      "434 8.440938836429268e-05\n",
      "435 8.294936560560018e-05\n",
      "436 8.116649405565113e-05\n",
      "437 7.977640052558854e-05\n",
      "438 7.855721196392551e-05\n",
      "439 7.720699068158865e-05\n",
      "440 7.590382301714271e-05\n",
      "441 7.44155840948224e-05\n",
      "442 7.323589670704678e-05\n",
      "443 7.206301233964041e-05\n",
      "444 7.118978828657418e-05\n",
      "445 6.967964873183519e-05\n",
      "446 6.846834003226832e-05\n",
      "447 6.763407145626843e-05\n",
      "448 6.663945532636717e-05\n",
      "449 6.55617332085967e-05\n",
      "450 6.422794103855267e-05\n",
      "451 6.33582312730141e-05\n",
      "452 6.253242463571951e-05\n",
      "453 6.162205681903288e-05\n",
      "454 6.07291076448746e-05\n",
      "455 5.956392487860285e-05\n",
      "456 5.8488978538662195e-05\n",
      "457 5.7479901443002746e-05\n",
      "458 5.6719218264333904e-05\n",
      "459 5.5952463299036026e-05\n",
      "460 5.498967948369682e-05\n",
      "461 5.3896470490144566e-05\n",
      "462 5.323524965206161e-05\n",
      "463 5.2605828386731446e-05\n",
      "464 5.170427175471559e-05\n",
      "465 5.1087085012113675e-05\n",
      "466 5.029559542890638e-05\n",
      "467 4.9687616410665214e-05\n",
      "468 4.8879621317610145e-05\n",
      "469 4.815722058992833e-05\n",
      "470 4.739791984320618e-05\n",
      "471 4.693820301326923e-05\n",
      "472 4.6439527068287134e-05\n",
      "473 4.554444967652671e-05\n",
      "474 4.483666270971298e-05\n",
      "475 4.4220676500117406e-05\n",
      "476 4.382433326099999e-05\n",
      "477 4.322805398260243e-05\n",
      "478 4.277825428289361e-05\n",
      "479 4.209626422380097e-05\n",
      "480 4.1751300159376115e-05\n",
      "481 4.1112500184681267e-05\n",
      "482 4.074410389875993e-05\n",
      "483 4.0096947486745194e-05\n",
      "484 3.939083035220392e-05\n",
      "485 3.885071782860905e-05\n",
      "486 3.8538346416316926e-05\n",
      "487 3.806702443398535e-05\n",
      "488 3.763230051845312e-05\n",
      "489 3.706221832544543e-05\n",
      "490 3.6591460229828954e-05\n",
      "491 3.615435343817808e-05\n",
      "492 3.5851426218869165e-05\n",
      "493 3.533398194122128e-05\n",
      "494 3.5058867069892585e-05\n",
      "495 3.4771735954564065e-05\n",
      "496 3.4339500416535884e-05\n",
      "497 3.386061143828556e-05\n",
      "498 3.354066939209588e-05\n",
      "499 3.3055050153052434e-05\n"
     ]
    }
   ],
   "source": [
    "# 可运行代码见本文件夹中的 two_layer_net_tensor.py\n",
    "import torch\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "# N是批大小； D_in 是输入维度；\n",
    "# H 是隐藏层维度； D_out 是输出维度\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 产生随机输入和输出数据\n",
    "x = torch.randn(N, D_in, device=device)\n",
    "y = torch.randn(N, D_out, device=device)\n",
    "\n",
    "# 随机初始化权重\n",
    "w1 = torch.randn(D_in, H, device=device)\n",
    "w2 = torch.randn(H, D_out, device=device)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # 前向传播：计算预测值y\n",
    "    h = x.mm(w1)\n",
    "    h_relu = h.clamp(min=0)\n",
    "    y_pred = h_relu.mm(w2)\n",
    "\n",
    "    # 计算并输出loss；loss是存储在PyTorch的tensor中的标量，维度是()(零维标量）；\n",
    "    # 我们使用loss.item()得到tensor中的纯python数值。\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(t, loss.item())\n",
    "\n",
    "    # 反向传播，计算w1、w2对loss的梯度\n",
    "    grad_y_pred = 2.0 * (y_pred - y)\n",
    "    grad_w2 = h_relu.t().mm(grad_y_pred)\n",
    "    grad_h_relu = grad_y_pred.mm(w2.t())\n",
    "    grad_h = grad_h_relu.clone()\n",
    "    grad_h[h < 0] = 0\n",
    "    grad_w1 = x.t().mm(grad_h)\n",
    "\n",
    "    # 使用梯度下降更新权重\n",
    "    w1 -= learning_rate * grad_w1\n",
    "    w2 -= learning_rate * grad_w2"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "data": {
      "text/plain": "array([[4, 1],\n       [2, 2]])"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = [[1, 0], [0, 1]]\n",
    "b = [[4, 1], [2, 2]]\n",
    "np.dot(a, b)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 39873296.0\n",
      "1 46295424.0\n",
      "2 57085252.0\n",
      "3 57263480.0\n",
      "4 38185216.0\n",
      "5 16286675.0\n",
      "6 5651057.0\n",
      "7 2543157.5\n",
      "8 1644872.625\n",
      "9 1272449.75\n",
      "10 1043899.5\n",
      "11 874373.0625\n",
      "12 740199.875\n",
      "13 631541.5\n",
      "14 542230.8125\n",
      "15 468234.65625\n",
      "16 406442.875\n",
      "17 354626.53125\n",
      "18 310718.59375\n",
      "19 273292.25\n",
      "20 241248.296875\n",
      "21 213688.296875\n",
      "22 189864.359375\n",
      "23 169176.53125\n",
      "24 151133.125\n",
      "25 135375.859375\n",
      "26 121529.5703125\n",
      "27 109328.15625\n",
      "28 98545.1875\n",
      "29 89005.234375\n",
      "30 80534.4375\n",
      "31 72996.9296875\n",
      "32 66268.6640625\n",
      "33 60247.515625\n",
      "34 54849.5234375\n",
      "35 50001.58203125\n",
      "36 45638.5390625\n",
      "37 41706.75390625\n",
      "38 38157.06640625\n",
      "39 34949.265625\n",
      "40 32044.10546875\n",
      "41 29409.25390625\n",
      "42 27017.30078125\n",
      "43 24841.341796875\n",
      "44 22860.27734375\n",
      "45 21054.71875\n",
      "46 19406.359375\n",
      "47 17900.267578125\n",
      "48 16523.267578125\n",
      "49 15263.021484375\n",
      "50 14107.0263671875\n",
      "51 13046.8349609375\n",
      "52 12073.8994140625\n",
      "53 11180.2490234375\n",
      "54 10358.48046875\n",
      "55 9602.45703125\n",
      "56 8906.212890625\n",
      "57 8264.6728515625\n",
      "58 7673.10205078125\n",
      "59 7127.67919921875\n",
      "60 6625.14599609375\n",
      "61 6160.8525390625\n",
      "62 5731.56396484375\n",
      "63 5334.501953125\n",
      "64 4966.90869140625\n",
      "65 4626.5390625\n",
      "66 4311.248046875\n",
      "67 4018.8779296875\n",
      "68 3747.83251953125\n",
      "69 3496.21142578125\n",
      "70 3262.549560546875\n",
      "71 3045.586181640625\n",
      "72 2843.95263671875\n",
      "73 2656.54345703125\n",
      "74 2482.230224609375\n",
      "75 2320.070068359375\n",
      "76 2169.135498046875\n",
      "77 2028.5797119140625\n",
      "78 1897.744384765625\n",
      "79 1776.1041259765625\n",
      "80 1662.7274169921875\n",
      "81 1556.9923095703125\n",
      "82 1458.36474609375\n",
      "83 1366.3087158203125\n",
      "84 1280.376220703125\n",
      "85 1200.1298828125\n",
      "86 1125.1878662109375\n",
      "87 1055.1885986328125\n",
      "88 989.767578125\n",
      "89 928.6260375976562\n",
      "90 871.4421997070312\n",
      "91 817.9400634765625\n",
      "92 767.8994750976562\n",
      "93 721.0599365234375\n",
      "94 677.2021484375\n",
      "95 636.1460571289062\n",
      "96 597.7073364257812\n",
      "97 561.6776733398438\n",
      "98 527.925537109375\n",
      "99 496.2860107421875\n",
      "100 466.6255798339844\n",
      "101 438.82061767578125\n",
      "102 412.739013671875\n",
      "103 388.28375244140625\n",
      "104 365.328125\n",
      "105 343.78887939453125\n",
      "106 323.57275390625\n",
      "107 304.5941162109375\n",
      "108 286.7642517089844\n",
      "109 270.0279235839844\n",
      "110 254.31155395507812\n",
      "111 239.54635620117188\n",
      "112 225.6654052734375\n",
      "113 212.62258911132812\n",
      "114 200.35890197753906\n",
      "115 188.82928466796875\n",
      "116 177.98715209960938\n",
      "117 167.79701232910156\n",
      "118 158.20933532714844\n",
      "119 149.20883178710938\n",
      "120 140.7412109375\n",
      "121 132.7766876220703\n",
      "122 125.27316284179688\n",
      "123 118.21173095703125\n",
      "124 111.55840301513672\n",
      "125 105.29605102539062\n",
      "126 99.39511108398438\n",
      "127 93.83661651611328\n",
      "128 88.59954833984375\n",
      "129 83.6633071899414\n",
      "130 79.01190185546875\n",
      "131 74.62840270996094\n",
      "132 70.4985122680664\n",
      "133 66.60093688964844\n",
      "134 62.92639923095703\n",
      "135 59.461429595947266\n",
      "136 56.19200134277344\n",
      "137 53.10734176635742\n",
      "138 50.19551467895508\n",
      "139 47.451324462890625\n",
      "140 44.860801696777344\n",
      "141 42.4161262512207\n",
      "142 40.10972595214844\n",
      "143 37.93141174316406\n",
      "144 35.874366760253906\n",
      "145 33.93202590942383\n",
      "146 32.09725570678711\n",
      "147 30.365049362182617\n",
      "148 28.728288650512695\n",
      "149 27.182464599609375\n",
      "150 25.722627639770508\n",
      "151 24.344039916992188\n",
      "152 23.04027557373047\n",
      "153 21.808551788330078\n",
      "154 20.6435489654541\n",
      "155 19.543245315551758\n",
      "156 18.50246810913086\n",
      "157 17.519243240356445\n",
      "158 16.589035034179688\n",
      "159 15.710244178771973\n",
      "160 14.877805709838867\n",
      "161 14.093070983886719\n",
      "162 13.348671913146973\n",
      "163 12.64478874206543\n",
      "164 11.978519439697266\n",
      "165 11.349639892578125\n",
      "166 10.753421783447266\n",
      "167 10.189248085021973\n",
      "168 9.656149864196777\n",
      "169 9.1511812210083\n",
      "170 8.673376083374023\n",
      "171 8.220788955688477\n",
      "172 7.792515277862549\n",
      "173 7.386277198791504\n",
      "174 7.002514362335205\n",
      "175 6.63905143737793\n",
      "176 6.2951507568359375\n",
      "177 5.968865394592285\n",
      "178 5.660014629364014\n",
      "179 5.368079662322998\n",
      "180 5.091157913208008\n",
      "181 4.828483581542969\n",
      "182 4.5800275802612305\n",
      "183 4.344159126281738\n",
      "184 4.121265411376953\n",
      "185 3.909883737564087\n",
      "186 3.70965838432312\n",
      "187 3.5196213722229004\n",
      "188 3.3393800258636475\n",
      "189 3.1688077449798584\n",
      "190 3.0071513652801514\n",
      "191 2.8540027141571045\n",
      "192 2.7086856365203857\n",
      "193 2.5706820487976074\n",
      "194 2.43989896774292\n",
      "195 2.3160924911499023\n",
      "196 2.1986114978790283\n",
      "197 2.087043523788452\n",
      "198 1.9812918901443481\n",
      "199 1.8812371492385864\n",
      "200 1.7860604524612427\n",
      "201 1.6960396766662598\n",
      "202 1.6105787754058838\n",
      "203 1.5293161869049072\n",
      "204 1.4524235725402832\n",
      "205 1.379316806793213\n",
      "206 1.310044527053833\n",
      "207 1.2443699836730957\n",
      "208 1.1819921731948853\n",
      "209 1.1227920055389404\n",
      "210 1.0666053295135498\n",
      "211 1.0133217573165894\n",
      "212 0.9627432823181152\n",
      "213 0.9145222306251526\n",
      "214 0.868845522403717\n",
      "215 0.8256617784500122\n",
      "216 0.7843969464302063\n",
      "217 0.745492696762085\n",
      "218 0.7083736062049866\n",
      "219 0.6732933521270752\n",
      "220 0.6396716833114624\n",
      "221 0.6079782247543335\n",
      "222 0.5779584646224976\n",
      "223 0.5492334961891174\n",
      "224 0.5220907330513\n",
      "225 0.49632713198661804\n",
      "226 0.47182711958885193\n",
      "227 0.4486261308193207\n",
      "228 0.42647504806518555\n",
      "229 0.40545785427093506\n",
      "230 0.3854115605354309\n",
      "231 0.3664684295654297\n",
      "232 0.3484877049922943\n",
      "233 0.3313717246055603\n",
      "234 0.3150685131549835\n",
      "235 0.29973235726356506\n",
      "236 0.2849700152873993\n",
      "237 0.2709977328777313\n",
      "238 0.25771743059158325\n",
      "239 0.2450818121433258\n",
      "240 0.2331140786409378\n",
      "241 0.22179564833641052\n",
      "242 0.21092462539672852\n",
      "243 0.20064303278923035\n",
      "244 0.19083306193351746\n",
      "245 0.18151482939720154\n",
      "246 0.17276214063167572\n",
      "247 0.16436627507209778\n",
      "248 0.1563342809677124\n",
      "249 0.14875991642475128\n",
      "250 0.14158670604228973\n",
      "251 0.13472901284694672\n",
      "252 0.12818874418735504\n",
      "253 0.12197559326887131\n",
      "254 0.11602304875850677\n",
      "255 0.11040911078453064\n",
      "256 0.10504943132400513\n",
      "257 0.09999240189790726\n",
      "258 0.0952044352889061\n",
      "259 0.09061412513256073\n",
      "260 0.08625362813472748\n",
      "261 0.08213172852993011\n",
      "262 0.07817006856203079\n",
      "263 0.07439684122800827\n",
      "264 0.07082325220108032\n",
      "265 0.06743834912776947\n",
      "266 0.06419536471366882\n",
      "267 0.06109572947025299\n",
      "268 0.058161377906799316\n",
      "269 0.05538370460271835\n",
      "270 0.05272053927183151\n",
      "271 0.050212256610393524\n",
      "272 0.047789402306079865\n",
      "273 0.045515283942222595\n",
      "274 0.04334647208452225\n",
      "275 0.041269347071647644\n",
      "276 0.039301544427871704\n",
      "277 0.037419140338897705\n",
      "278 0.03562675043940544\n",
      "279 0.03393970802426338\n",
      "280 0.03234896808862686\n",
      "281 0.030802644789218903\n",
      "282 0.029334012418985367\n",
      "283 0.027928099036216736\n",
      "284 0.026609385386109352\n",
      "285 0.02535957470536232\n",
      "286 0.024148670956492424\n",
      "287 0.023003671318292618\n",
      "288 0.02191663347184658\n",
      "289 0.020901311188936234\n",
      "290 0.019916251301765442\n",
      "291 0.018981367349624634\n",
      "292 0.018080882728099823\n",
      "293 0.017249535769224167\n",
      "294 0.016449719667434692\n",
      "295 0.01567091792821884\n",
      "296 0.014935784041881561\n",
      "297 0.014237513765692711\n",
      "298 0.01357276737689972\n",
      "299 0.012951667420566082\n",
      "300 0.012352983467280865\n",
      "301 0.011786523275077343\n",
      "302 0.011236697435379028\n",
      "303 0.010724048130214214\n",
      "304 0.010230167768895626\n",
      "305 0.009765051305294037\n",
      "306 0.009317904710769653\n",
      "307 0.008886563591659069\n",
      "308 0.008489190600812435\n",
      "309 0.008099749684333801\n",
      "310 0.007735937833786011\n",
      "311 0.007384825497865677\n",
      "312 0.007046903949230909\n",
      "313 0.006727846339344978\n",
      "314 0.006420092657208443\n",
      "315 0.006132686976343393\n",
      "316 0.005859988275915384\n",
      "317 0.0056006149388849735\n",
      "318 0.005354462657123804\n",
      "319 0.005118083208799362\n",
      "320 0.004898404702544212\n",
      "321 0.004678080324083567\n",
      "322 0.0044715991243720055\n",
      "323 0.004277234431356192\n",
      "324 0.004093348979949951\n",
      "325 0.003916354384273291\n",
      "326 0.0037538795731961727\n",
      "327 0.0035876359324902296\n",
      "328 0.0034430567175149918\n",
      "329 0.0032943272963166237\n",
      "330 0.003157435217872262\n",
      "331 0.003025854006409645\n",
      "332 0.002899615094065666\n",
      "333 0.0027770106680691242\n",
      "334 0.0026600405108183622\n",
      "335 0.002552164252847433\n",
      "336 0.002447605598717928\n",
      "337 0.002347879111766815\n",
      "338 0.0022518420591950417\n",
      "339 0.0021640113554894924\n",
      "340 0.002078615128993988\n",
      "341 0.0019929015543311834\n",
      "342 0.0019159173825755715\n",
      "343 0.0018409235635772347\n",
      "344 0.0017700776224955916\n",
      "345 0.001700438093394041\n",
      "346 0.0016358232824131846\n",
      "347 0.0015766604337841272\n",
      "348 0.0015163837233558297\n",
      "349 0.001458754763007164\n",
      "350 0.001403465517796576\n",
      "351 0.0013522637309506536\n",
      "352 0.0013010536786168814\n",
      "353 0.0012537541333585978\n",
      "354 0.0012089646188542247\n",
      "355 0.0011677193688228726\n",
      "356 0.0011265611974522471\n",
      "357 0.0010847784578800201\n",
      "358 0.0010470915585756302\n",
      "359 0.0010119613725692034\n",
      "360 0.0009752860642038286\n",
      "361 0.0009426093893125653\n",
      "362 0.0009100465103983879\n",
      "363 0.0008802533848211169\n",
      "364 0.0008509072358720005\n",
      "365 0.0008222903707064688\n",
      "366 0.0007957264315336943\n",
      "367 0.0007689512567594647\n",
      "368 0.000743841752409935\n",
      "369 0.0007206191075965762\n",
      "370 0.0006976042641326785\n",
      "371 0.0006764050340279937\n",
      "372 0.0006548546953126788\n",
      "373 0.0006331250187940896\n",
      "374 0.0006141349440440536\n",
      "375 0.0005937782698310912\n",
      "376 0.000576543272472918\n",
      "377 0.0005591884255409241\n",
      "378 0.0005421097157523036\n",
      "379 0.0005252761766314507\n",
      "380 0.0005087880417704582\n",
      "381 0.0004948501009494066\n",
      "382 0.000480331655126065\n",
      "383 0.00046476349234580994\n",
      "384 0.0004518497153185308\n",
      "385 0.00044004266965202987\n",
      "386 0.00042841891990974545\n",
      "387 0.0004158100637141615\n",
      "388 0.0004050309653393924\n",
      "389 0.0003939415037166327\n",
      "390 0.00038320390740409493\n",
      "391 0.00037281197728589177\n",
      "392 0.0003628545382525772\n",
      "393 0.0003537628217600286\n",
      "394 0.000343221181537956\n",
      "395 0.00033462868304923177\n",
      "396 0.0003258574288338423\n",
      "397 0.00031686737202107906\n",
      "398 0.00030874411459080875\n",
      "399 0.00030188134405761957\n",
      "400 0.0002932533388957381\n",
      "401 0.00028519888292066753\n",
      "402 0.0002787404810078442\n",
      "403 0.00027129624504595995\n",
      "404 0.00026565443840809166\n",
      "405 0.00025864189956337214\n",
      "406 0.0002519121626392007\n",
      "407 0.00024532669340260327\n",
      "408 0.00023942960251588374\n",
      "409 0.0002338970371056348\n",
      "410 0.00022795246331952512\n",
      "411 0.00022256853117141873\n",
      "412 0.00021760135132353753\n",
      "413 0.00021196655870880932\n",
      "414 0.00020696927094832063\n",
      "415 0.00020221453451085836\n",
      "416 0.00019782439630944282\n",
      "417 0.00019356410484761\n",
      "418 0.00018967900541611016\n",
      "419 0.00018501201702747494\n",
      "420 0.00018070248188450933\n",
      "421 0.00017646556079853326\n",
      "422 0.0001726782793411985\n",
      "423 0.00016925748786889017\n",
      "424 0.00016561457596253604\n",
      "425 0.00016201373364310712\n",
      "426 0.00015872523363213986\n",
      "427 0.00015530793461948633\n",
      "428 0.00015200361667666584\n",
      "429 0.00014895467029418796\n",
      "430 0.0001461320061935112\n",
      "431 0.0001432788703823462\n",
      "432 0.00014057916996534914\n",
      "433 0.00013758105342276394\n",
      "434 0.0001345444325124845\n",
      "435 0.00013212685007601976\n",
      "436 0.0001294098619837314\n",
      "437 0.00012735737254843116\n",
      "438 0.00012468741624616086\n",
      "439 0.00012192900612717494\n",
      "440 0.00011985332821495831\n",
      "441 0.00011736661690520123\n",
      "442 0.00011544410517672077\n",
      "443 0.00011369849380571395\n",
      "444 0.00011126918252557516\n",
      "445 0.00010931114957202226\n",
      "446 0.00010721058788476512\n",
      "447 0.00010527411359362304\n",
      "448 0.00010328850476071239\n",
      "449 0.00010136993660125881\n",
      "450 9.950138337444514e-05\n",
      "451 9.77211821009405e-05\n",
      "452 9.611077985027805e-05\n",
      "453 9.459479042561725e-05\n",
      "454 9.295557538280264e-05\n",
      "455 9.125931683229282e-05\n",
      "456 8.973729563876987e-05\n",
      "457 8.855819760356098e-05\n",
      "458 8.691601397003978e-05\n",
      "459 8.557868568459526e-05\n",
      "460 8.382750820601359e-05\n",
      "461 8.230652019847184e-05\n",
      "462 8.114107913570479e-05\n",
      "463 7.992421160452068e-05\n",
      "464 7.883338548708707e-05\n",
      "465 7.731624646112323e-05\n",
      "466 7.60370385251008e-05\n",
      "467 7.511612784583122e-05\n",
      "468 7.403932977467775e-05\n",
      "469 7.303083839360625e-05\n",
      "470 7.165675924625248e-05\n",
      "471 7.033341535134241e-05\n",
      "472 6.925116031197831e-05\n",
      "473 6.821267743362114e-05\n",
      "474 6.747255247319117e-05\n",
      "475 6.643972301390022e-05\n",
      "476 6.533051055157557e-05\n",
      "477 6.457616836996749e-05\n",
      "478 6.348975875880569e-05\n",
      "479 6.270351877901703e-05\n",
      "480 6.176369060995057e-05\n",
      "481 6.085662607802078e-05\n",
      "482 6.0168051277287304e-05\n",
      "483 5.9435271396068856e-05\n",
      "484 5.8494144468568265e-05\n",
      "485 5.7583500165492296e-05\n",
      "486 5.668653466273099e-05\n",
      "487 5.585396502283402e-05\n",
      "488 5.488160968525335e-05\n",
      "489 5.439410961116664e-05\n",
      "490 5.3692525398219004e-05\n",
      "491 5.29424287378788e-05\n",
      "492 5.22567679581698e-05\n",
      "493 5.169279393157922e-05\n",
      "494 5.085978773422539e-05\n",
      "495 5.008559310226701e-05\n",
      "496 4.9764694267651066e-05\n",
      "497 4.924758832203224e-05\n",
      "498 4.8697351303417236e-05\n",
      "499 4.794357664650306e-05\n"
     ]
    }
   ],
   "source": [
    "# 可运行代码见本文件夹中的 two_layer_net_autograd.py\n",
    "import torch\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "# N是批大小；D_in是输入维度；\n",
    "# H是隐藏层维度；D_out是输出维度\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 产生随机输入和输出数据\n",
    "x = torch.randn(N, D_in, device=device)\n",
    "y = torch.randn(N, D_out, device=device)\n",
    "\n",
    "# 产生随机权重tensor，将requires_grad设置为True意味着我们希望在反向传播时候计算这些值的梯度\n",
    "w1 = torch.randn(D_in, H, device=device, requires_grad=True)\n",
    "w2 = torch.randn(H, D_out, device=device, requires_grad=True)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "\n",
    "    # 前向传播：使用tensor的操作计算预测值y。\n",
    "    # 由于w1和w2有requires_grad=True，涉及这些张量的操作将让PyTorch构建计算图，\n",
    "    # 从而允许自动计算梯度。由于我们不再手工实现反向传播，所以不需要保留中间值的引用。\n",
    "    y_pred = x.mm(w1).clamp(min=0).mm(w2)\n",
    "\n",
    "    # 计算并输出loss，loss是一个形状为()的张量，loss.item()是这个张量对应的python数值\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(t, loss.item())\n",
    "\n",
    "    # 使用autograd计算反向传播。这个调用将计算loss对所有requires_grad=True的tensor的梯度。\n",
    "    # 这次调用后，w1.grad和w2.grad将分别是loss对w1和w2的梯度张量。\n",
    "    loss.backward()\n",
    "\n",
    "\n",
    "    # 使用梯度下降更新权重。对于这一步，我们只想对w1和w2的值进行原地改变；不想为更新阶段构建计算图，\n",
    "    # 所以我们使用torch.no_grad()上下文管理器防止PyTorch为更新构建计算图\n",
    "    with torch.no_grad():\n",
    "        w1 -= learning_rate * w1.grad\n",
    "        w2 -= learning_rate * w2.grad\n",
    "\n",
    "        # 反向传播之后手动置零梯度\n",
    "        w1.grad.zero_()\n",
    "        w2.grad.zero_()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 32839518.0\n",
      "1 29487572.0\n",
      "2 27223510.0\n",
      "3 22918276.0\n",
      "4 16799184.0\n",
      "5 10788627.0\n",
      "6 6437140.5\n",
      "7 3814017.5\n",
      "8 2381135.75\n",
      "9 1609010.75\n",
      "10 1177525.375\n",
      "11 916826.75\n",
      "12 745200.125\n",
      "13 622692.4375\n",
      "14 529683.0625\n",
      "15 456018.625\n",
      "16 395803.25\n",
      "17 345920.0\n",
      "18 304117.4375\n",
      "19 268515.4375\n",
      "20 237898.40625\n",
      "21 211525.71875\n",
      "22 188710.25\n",
      "23 168851.984375\n",
      "24 151494.875\n",
      "25 136285.625\n",
      "26 122918.921875\n",
      "27 111110.828125\n",
      "28 100638.28125\n",
      "29 91334.53125\n",
      "30 83045.375\n",
      "31 75645.4375\n",
      "32 69010.9921875\n",
      "33 63073.40625\n",
      "34 57722.375\n",
      "35 52896.84375\n",
      "36 48536.74609375\n",
      "37 44590.9765625\n",
      "38 41010.2578125\n",
      "39 37758.359375\n",
      "40 34807.19921875\n",
      "41 32118.8359375\n",
      "42 29666.765625\n",
      "43 27427.19140625\n",
      "44 25380.453125\n",
      "45 23509.15625\n",
      "46 21792.650390625\n",
      "47 20216.775390625\n",
      "48 18767.771484375\n",
      "49 17435.09375\n",
      "50 16207.736328125\n",
      "51 15076.625\n",
      "52 14032.6591796875\n",
      "53 13070.48828125\n",
      "54 12182.154296875\n",
      "55 11360.3359375\n",
      "56 10600.1708984375\n",
      "57 9895.3017578125\n",
      "58 9241.634765625\n",
      "59 8635.4228515625\n",
      "60 8072.703125\n",
      "61 7550.17578125\n",
      "62 7064.533203125\n",
      "63 6613.1201171875\n",
      "64 6193.27392578125\n",
      "65 5802.18896484375\n",
      "66 5437.89111328125\n",
      "67 5098.47314453125\n",
      "68 4781.83837890625\n",
      "69 4486.41748046875\n",
      "70 4210.6865234375\n",
      "71 3953.1923828125\n",
      "72 3712.71240234375\n",
      "73 3488.120849609375\n",
      "74 3278.046630859375\n",
      "75 3081.484130859375\n",
      "76 2897.5234375\n",
      "77 2725.40380859375\n",
      "78 2564.37109375\n",
      "79 2413.576904296875\n",
      "80 2272.27880859375\n",
      "81 2139.777587890625\n",
      "82 2015.512451171875\n",
      "83 1898.952392578125\n",
      "84 1789.503173828125\n",
      "85 1686.8206787109375\n",
      "86 1590.4036865234375\n",
      "87 1499.823486328125\n",
      "88 1414.6917724609375\n",
      "89 1334.673828125\n",
      "90 1259.436279296875\n",
      "91 1188.700927734375\n",
      "92 1122.1639404296875\n",
      "93 1059.588134765625\n",
      "94 1000.642578125\n",
      "95 945.1824951171875\n",
      "96 892.9614868164062\n",
      "97 843.7432861328125\n",
      "98 797.3880615234375\n",
      "99 753.7256469726562\n",
      "100 712.5704345703125\n",
      "101 673.76806640625\n",
      "102 637.1859130859375\n",
      "103 602.6841430664062\n",
      "104 570.1409912109375\n",
      "105 539.4458618164062\n",
      "106 510.47100830078125\n",
      "107 483.12347412109375\n",
      "108 457.3069763183594\n",
      "109 432.92822265625\n",
      "110 409.91107177734375\n",
      "111 388.1763916015625\n",
      "112 367.6567077636719\n",
      "113 348.252685546875\n",
      "114 329.9115295410156\n",
      "115 312.5769348144531\n",
      "116 296.1944274902344\n",
      "117 280.699462890625\n",
      "118 266.0484619140625\n",
      "119 252.19223022460938\n",
      "120 239.08584594726562\n",
      "121 226.6962127685547\n",
      "122 214.9632568359375\n",
      "123 203.8646240234375\n",
      "124 193.35662841796875\n",
      "125 183.40908813476562\n",
      "126 173.99179077148438\n",
      "127 165.07089233398438\n",
      "128 156.63113403320312\n",
      "129 148.6320343017578\n",
      "130 141.06045532226562\n",
      "131 133.88844299316406\n",
      "132 127.09159851074219\n",
      "133 120.6507568359375\n",
      "134 114.55445098876953\n",
      "135 108.767822265625\n",
      "136 103.28941345214844\n",
      "137 98.09320831298828\n",
      "138 93.16679382324219\n",
      "139 88.49272155761719\n",
      "140 84.06275177001953\n",
      "141 79.86080932617188\n",
      "142 75.87547302246094\n",
      "143 72.09268951416016\n",
      "144 68.5066909790039\n",
      "145 65.10245513916016\n",
      "146 61.870635986328125\n",
      "147 58.80695724487305\n",
      "148 55.896217346191406\n",
      "149 53.13530349731445\n",
      "150 50.51490020751953\n",
      "151 48.02488708496094\n",
      "152 45.661109924316406\n",
      "153 43.41691589355469\n",
      "154 41.2850341796875\n",
      "155 39.26329040527344\n",
      "156 37.33860397338867\n",
      "157 35.512535095214844\n",
      "158 33.77927780151367\n",
      "159 32.13191604614258\n",
      "160 30.566749572753906\n",
      "161 29.07904815673828\n",
      "162 27.666162490844727\n",
      "163 26.321949005126953\n",
      "164 25.04486656188965\n",
      "165 23.831554412841797\n",
      "166 22.679487228393555\n",
      "167 21.581769943237305\n",
      "168 20.540481567382812\n",
      "169 19.549915313720703\n",
      "170 18.607032775878906\n",
      "171 17.71090316772461\n",
      "172 16.85868263244629\n",
      "173 16.048547744750977\n",
      "174 15.279814720153809\n",
      "175 14.54621410369873\n",
      "176 13.848869323730469\n",
      "177 13.186777114868164\n",
      "178 12.55605697631836\n",
      "179 11.95528507232666\n",
      "180 11.385801315307617\n",
      "181 10.841994285583496\n",
      "182 10.325054168701172\n",
      "183 9.83326530456543\n",
      "184 9.365513801574707\n",
      "185 8.920166015625\n",
      "186 8.49632740020752\n",
      "187 8.093094825744629\n",
      "188 7.710196495056152\n",
      "189 7.344766616821289\n",
      "190 6.996491432189941\n",
      "191 6.665830135345459\n",
      "192 6.350821018218994\n",
      "193 6.049724102020264\n",
      "194 5.764510631561279\n",
      "195 5.492558002471924\n",
      "196 5.23405647277832\n",
      "197 4.987741470336914\n",
      "198 4.7527642250061035\n",
      "199 4.52911901473999\n",
      "200 4.315844535827637\n",
      "201 4.112972259521484\n",
      "202 3.9201178550720215\n",
      "203 3.7363152503967285\n",
      "204 3.5610952377319336\n",
      "205 3.394397735595703\n",
      "206 3.2356836795806885\n",
      "207 3.084066152572632\n",
      "208 2.9398727416992188\n",
      "209 2.802626371383667\n",
      "210 2.671351194381714\n",
      "211 2.5469110012054443\n",
      "212 2.427926540374756\n",
      "213 2.31463623046875\n",
      "214 2.2071192264556885\n",
      "215 2.104214668273926\n",
      "216 2.0057268142700195\n",
      "217 1.9127697944641113\n",
      "218 1.823790192604065\n",
      "219 1.7391043901443481\n",
      "220 1.6581857204437256\n",
      "221 1.581296443939209\n",
      "222 1.5078234672546387\n",
      "223 1.4379665851593018\n",
      "224 1.3714773654937744\n",
      "225 1.3077448606491089\n",
      "226 1.2472646236419678\n",
      "227 1.1895349025726318\n",
      "228 1.1346522569656372\n",
      "229 1.0819374322891235\n",
      "230 1.0320501327514648\n",
      "231 0.9842655658721924\n",
      "232 0.9387724995613098\n",
      "233 0.8955816030502319\n",
      "234 0.8542330265045166\n",
      "235 0.8149703741073608\n",
      "236 0.7774012088775635\n",
      "237 0.7414720058441162\n",
      "238 0.7073192596435547\n",
      "239 0.6748520731925964\n",
      "240 0.6437660455703735\n",
      "241 0.6142287254333496\n",
      "242 0.5859352946281433\n",
      "243 0.5589460730552673\n",
      "244 0.5332191586494446\n",
      "245 0.5088461637496948\n",
      "246 0.48545658588409424\n",
      "247 0.46315330266952515\n",
      "248 0.44188109040260315\n",
      "249 0.4216374158859253\n",
      "250 0.40235406160354614\n",
      "251 0.3839085102081299\n",
      "252 0.36634671688079834\n",
      "253 0.3495449125766754\n",
      "254 0.33357685804367065\n",
      "255 0.31835031509399414\n",
      "256 0.3037022054195404\n",
      "257 0.28975099325180054\n",
      "258 0.2765038311481476\n",
      "259 0.26393136382102966\n",
      "260 0.2519514262676239\n",
      "261 0.24039243161678314\n",
      "262 0.22948206961154938\n",
      "263 0.2189638763666153\n",
      "264 0.20903664827346802\n",
      "265 0.19953110814094543\n",
      "266 0.19041001796722412\n",
      "267 0.18176396191120148\n",
      "268 0.17345896363258362\n",
      "269 0.16560854017734528\n",
      "270 0.15803572535514832\n",
      "271 0.1508214771747589\n",
      "272 0.14398935437202454\n",
      "273 0.13740426301956177\n",
      "274 0.13116984069347382\n",
      "275 0.1252482682466507\n",
      "276 0.11956124752759933\n",
      "277 0.11414240300655365\n",
      "278 0.10894755274057388\n",
      "279 0.1039639264345169\n",
      "280 0.09921294450759888\n",
      "281 0.0947204977273941\n",
      "282 0.09046683460474014\n",
      "283 0.08635703474283218\n",
      "284 0.08247044682502747\n",
      "285 0.07872400432825089\n",
      "286 0.07516982406377792\n",
      "287 0.07178519666194916\n",
      "288 0.06851349771022797\n",
      "289 0.06541308015584946\n",
      "290 0.062478456646203995\n",
      "291 0.059634946286678314\n",
      "292 0.05694683641195297\n",
      "293 0.054355934262275696\n",
      "294 0.05193021893501282\n",
      "295 0.04957713186740875\n",
      "296 0.04733061417937279\n",
      "297 0.045201048254966736\n",
      "298 0.043173111975193024\n",
      "299 0.041264235973358154\n",
      "300 0.03938763961195946\n",
      "301 0.03760967031121254\n",
      "302 0.035943638533353806\n",
      "303 0.03432183340191841\n",
      "304 0.03276319429278374\n",
      "305 0.03128840774297714\n",
      "306 0.02988235466182232\n",
      "307 0.02854216657578945\n",
      "308 0.02726839669048786\n",
      "309 0.026022881269454956\n",
      "310 0.02486375719308853\n",
      "311 0.023773733526468277\n",
      "312 0.0227062925696373\n",
      "313 0.021696796640753746\n",
      "314 0.02074243500828743\n",
      "315 0.01981961727142334\n",
      "316 0.018945395946502686\n",
      "317 0.01810828596353531\n",
      "318 0.0173112191259861\n",
      "319 0.0165188480168581\n",
      "320 0.01578066125512123\n",
      "321 0.015081193298101425\n",
      "322 0.01441088318824768\n",
      "323 0.013753057457506657\n",
      "324 0.013165170326828957\n",
      "325 0.012591327540576458\n",
      "326 0.012040081433951855\n",
      "327 0.011504344642162323\n",
      "328 0.011003432795405388\n",
      "329 0.010522239841520786\n",
      "330 0.010065019130706787\n",
      "331 0.009624326601624489\n",
      "332 0.009202653542160988\n",
      "333 0.008802035823464394\n",
      "334 0.008422049693763256\n",
      "335 0.008059872314333916\n",
      "336 0.007714464329183102\n",
      "337 0.0073789553716778755\n",
      "338 0.007066346239298582\n",
      "339 0.006765306927263737\n",
      "340 0.006471665110439062\n",
      "341 0.006191832013428211\n",
      "342 0.005934476852416992\n",
      "343 0.0056784194894135\n",
      "344 0.005433616228401661\n",
      "345 0.005197232123464346\n",
      "346 0.004983674734830856\n",
      "347 0.0047707874327898026\n",
      "348 0.004574798513203859\n",
      "349 0.0043809013441205025\n",
      "350 0.004201989620923996\n",
      "351 0.004032088443636894\n",
      "352 0.003866287413984537\n",
      "353 0.0037043520715087652\n",
      "354 0.0035526014398783445\n",
      "355 0.0034081165213137865\n",
      "356 0.0032695657573640347\n",
      "357 0.0031385670881718397\n",
      "358 0.0030093910172581673\n",
      "359 0.00288993283174932\n",
      "360 0.002777678892016411\n",
      "361 0.0026691085658967495\n",
      "362 0.0025670963805168867\n",
      "363 0.0024692737497389317\n",
      "364 0.0023749200627207756\n",
      "365 0.0022831258829683065\n",
      "366 0.0021933377720415592\n",
      "367 0.002108885208144784\n",
      "368 0.002028071554377675\n",
      "369 0.0019500406924635172\n",
      "370 0.0018771005561575294\n",
      "371 0.0018072286620736122\n",
      "372 0.0017389771528542042\n",
      "373 0.0016718883998692036\n",
      "374 0.0016097447369247675\n",
      "375 0.0015521518653258681\n",
      "376 0.0014950695913285017\n",
      "377 0.0014410455478355289\n",
      "378 0.0013885337393730879\n",
      "379 0.0013397005386650562\n",
      "380 0.0012921540765091777\n",
      "381 0.0012480714358389378\n",
      "382 0.0012067030183970928\n",
      "383 0.0011639437871053815\n",
      "384 0.001124791568145156\n",
      "385 0.0010826269863173366\n",
      "386 0.0010468290420249104\n",
      "387 0.0010129479924216866\n",
      "388 0.0009787615854293108\n",
      "389 0.0009443626622669399\n",
      "390 0.0009128916426561773\n",
      "391 0.00088428397430107\n",
      "392 0.0008566822507418692\n",
      "393 0.0008292609127238393\n",
      "394 0.0008021170506253839\n",
      "395 0.0007748123025521636\n",
      "396 0.0007521788356825709\n",
      "397 0.0007290412904694676\n",
      "398 0.0007050889544188976\n",
      "399 0.0006816855748184025\n",
      "400 0.0006610972341150045\n",
      "401 0.0006425873725675046\n",
      "402 0.0006230946164578199\n",
      "403 0.0006039663567207754\n",
      "404 0.0005854953778907657\n",
      "405 0.0005666576325893402\n",
      "406 0.0005497899255715311\n",
      "407 0.0005337127367965877\n",
      "408 0.0005188904469832778\n",
      "409 0.0005035663489252329\n",
      "410 0.0004889973206445575\n",
      "411 0.0004748714272864163\n",
      "412 0.00046048464719206095\n",
      "413 0.000448214472271502\n",
      "414 0.00043649456347338855\n",
      "415 0.00042431021574884653\n",
      "416 0.000411369459470734\n",
      "417 0.00040068908128887415\n",
      "418 0.00039049331098794937\n",
      "419 0.00037976267049089074\n",
      "420 0.0003707101568579674\n",
      "421 0.0003602822544053197\n",
      "422 0.00035109181771986187\n",
      "423 0.0003420788561925292\n",
      "424 0.0003328665334265679\n",
      "425 0.00032503693364560604\n",
      "426 0.000316482299240306\n",
      "427 0.00030842237174510956\n",
      "428 0.0003000354627147317\n",
      "429 0.00029289210215210915\n",
      "430 0.0002857687941286713\n",
      "431 0.0002784321259241551\n",
      "432 0.00027157244039699435\n",
      "433 0.00026550726033747196\n",
      "434 0.00025829795049503446\n",
      "435 0.0002525810559745878\n",
      "436 0.0002472198102623224\n",
      "437 0.0002411826717434451\n",
      "438 0.00023612419317942113\n",
      "439 0.00022992727463133633\n",
      "440 0.00022473478747997433\n",
      "441 0.00021922944870311767\n",
      "442 0.00021490611834451556\n",
      "443 0.0002102845028275624\n",
      "444 0.00020552381465677172\n",
      "445 0.0002011392207350582\n",
      "446 0.00019658508244901896\n",
      "447 0.00019243518181610852\n",
      "448 0.00018835769151337445\n",
      "449 0.00018391304183751345\n",
      "450 0.00018027830810751766\n",
      "451 0.00017638163990341127\n",
      "452 0.00017321949417237192\n",
      "453 0.00016943126684054732\n",
      "454 0.0001658228866290301\n",
      "455 0.0001624369469936937\n",
      "456 0.0001597806840436533\n",
      "457 0.00015619727491866797\n",
      "458 0.00015320834063459188\n",
      "459 0.00015019842248875648\n",
      "460 0.00014695573190692812\n",
      "461 0.00014421221567317843\n",
      "462 0.00014143848966341466\n",
      "463 0.00013840524479746819\n",
      "464 0.000135613459860906\n",
      "465 0.00013338029384613037\n",
      "466 0.00013056723400950432\n",
      "467 0.00012818865070585161\n",
      "468 0.0001257658877875656\n",
      "469 0.00012391468044370413\n",
      "470 0.00012166077794972807\n",
      "471 0.00011942956916755065\n",
      "472 0.00011703894415404648\n",
      "473 0.00011488554446259513\n",
      "474 0.00011314978473819792\n",
      "475 0.00011100984556833282\n",
      "476 0.00010902136273216456\n",
      "477 0.0001072288941941224\n",
      "478 0.00010511069558560848\n",
      "479 0.0001033442240441218\n",
      "480 0.00010127868154086173\n",
      "481 9.971541294362396e-05\n",
      "482 9.777462400961667e-05\n",
      "483 9.62782942224294e-05\n",
      "484 9.465744369663298e-05\n",
      "485 9.284260886488482e-05\n",
      "486 9.12881369004026e-05\n",
      "487 8.97446007002145e-05\n",
      "488 8.825222903396934e-05\n",
      "489 8.711942791705951e-05\n",
      "490 8.580838766647503e-05\n",
      "491 8.409141446463764e-05\n",
      "492 8.300453919218853e-05\n",
      "493 8.155575051205233e-05\n",
      "494 8.025850547710434e-05\n",
      "495 7.896502211224288e-05\n",
      "496 7.778638246236369e-05\n",
      "497 7.664060103707016e-05\n",
      "498 7.560329686384648e-05\n",
      "499 7.437447493430227e-05\n"
     ]
    }
   ],
   "source": [
    "# 可运行代码见本文件夹中的 two_layer_net_custom_function.py\n",
    "import torch\n",
    "\n",
    "class MyReLU(torch.autograd.Function):\n",
    "    \"\"\"\n",
    "    我们可以通过建立torch.autograd的子类来实现我们自定义的autograd函数，\n",
    "    并完成张量的正向和反向传播。\n",
    "    \"\"\"\n",
    "    @staticmethod\n",
    "    def forward(ctx, x):\n",
    "        \"\"\"\n",
    "        在正向传播中，我们接收到一个上下文对象和一个包含输入的张量；\n",
    "        我们必须返回一个包含输出的张量，\n",
    "        并且我们可以使用上下文对象来缓存对象，以便在反向传播中使用。\n",
    "        \"\"\"\n",
    "        ctx.save_for_backward(x)\n",
    "        return x.clamp(min=0)\n",
    "\n",
    "    @staticmethod\n",
    "    def backward(ctx, grad_output):\n",
    "        \"\"\"\n",
    "        在反向传播中，我们接收到上下文对象和一个张量，\n",
    "        其包含了相对于正向传播过程中产生的输出的损失的梯度。\n",
    "        我们可以从上下文对象中检索缓存的数据，\n",
    "        并且必须计算并返回与正向传播的输入相关的损失的梯度。\n",
    "        \"\"\"\n",
    "        x, = ctx.saved_tensors\n",
    "        grad_x = grad_output.clone()\n",
    "        grad_x[x < 0] = 0\n",
    "        return grad_x\n",
    "\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "# N是批大小； D_in 是输入维度；\n",
    "# H 是隐藏层维度； D_out 是输出维度\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 产生输入和输出的随机张量\n",
    "x = torch.randn(N, D_in, device=device)\n",
    "y = torch.randn(N, D_out, device=device)\n",
    "\n",
    "# 产生随机权重的张量\n",
    "w1 = torch.randn(D_in, H, device=device, requires_grad=True)\n",
    "w2 = torch.randn(H, D_out, device=device, requires_grad=True)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # 正向传播：使用张量上的操作来计算输出值y；\n",
    "    # 我们通过调用 MyReLU.apply 函数来使用自定义的ReLU\n",
    "    y_pred = MyReLU.apply(x.mm(w1)).mm(w2)\n",
    "\n",
    "    # 计算并输出loss\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(t, loss.item())\n",
    "\n",
    "    # 使用autograd计算反向传播过程。\n",
    "    loss.backward()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        # 用梯度下降更新权重\n",
    "        w1 -= learning_rate * w1.grad\n",
    "        w2 -= learning_rate * w2.grad\n",
    "\n",
    "        # 在反向传播之后手动清零梯度\n",
    "        w1.grad.zero_()\n",
    "        w2.grad.zero_()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 706.5322265625\n",
      "1 657.553955078125\n",
      "2 615.186279296875\n",
      "3 577.9329833984375\n",
      "4 544.5027465820312\n",
      "5 514.1419067382812\n",
      "6 486.3369140625\n",
      "7 460.68951416015625\n",
      "8 436.99334716796875\n",
      "9 414.85968017578125\n",
      "10 393.9879455566406\n",
      "11 374.2057800292969\n",
      "12 355.44781494140625\n",
      "13 337.5696716308594\n",
      "14 320.60028076171875\n",
      "15 304.3428649902344\n",
      "16 288.788818359375\n",
      "17 273.8682861328125\n",
      "18 259.56219482421875\n",
      "19 245.91831970214844\n",
      "20 232.869873046875\n",
      "21 220.36822509765625\n",
      "22 208.4119873046875\n",
      "23 196.90228271484375\n",
      "24 185.88888549804688\n",
      "25 175.3277130126953\n",
      "26 165.25038146972656\n",
      "27 155.65933227539062\n",
      "28 146.54232788085938\n",
      "29 137.87493896484375\n",
      "30 129.6371612548828\n",
      "31 121.83660888671875\n",
      "32 114.4262924194336\n",
      "33 107.4298095703125\n",
      "34 100.8284912109375\n",
      "35 94.60575103759766\n",
      "36 88.71762084960938\n",
      "37 83.1695327758789\n",
      "38 77.9605484008789\n",
      "39 73.06526947021484\n",
      "40 68.47437286376953\n",
      "41 64.16352844238281\n",
      "42 60.12721252441406\n",
      "43 56.35293197631836\n",
      "44 52.8154182434082\n",
      "45 49.49249267578125\n",
      "46 46.381038665771484\n",
      "47 43.46602249145508\n",
      "48 40.74245834350586\n",
      "49 38.203147888183594\n",
      "50 35.83479309082031\n",
      "51 33.61897659301758\n",
      "52 31.55205726623535\n",
      "53 29.624387741088867\n",
      "54 27.822189331054688\n",
      "55 26.13935089111328\n",
      "56 24.566181182861328\n",
      "57 23.09271240234375\n",
      "58 21.712329864501953\n",
      "59 20.423315048217773\n",
      "60 19.217323303222656\n",
      "61 18.08196449279785\n",
      "62 17.0194034576416\n",
      "63 16.024864196777344\n",
      "64 15.09355354309082\n",
      "65 14.221538543701172\n",
      "66 13.405821800231934\n",
      "67 12.640958786010742\n",
      "68 11.923759460449219\n",
      "69 11.251423835754395\n",
      "70 10.620865821838379\n",
      "71 10.028351783752441\n",
      "72 9.472535133361816\n",
      "73 8.950514793395996\n",
      "74 8.459859848022461\n",
      "75 7.999111175537109\n",
      "76 7.56532096862793\n",
      "77 7.1565260887146\n",
      "78 6.772446632385254\n",
      "79 6.411396026611328\n",
      "80 6.071381092071533\n",
      "81 5.7507758140563965\n",
      "82 5.448602676391602\n",
      "83 5.164609909057617\n",
      "84 4.8965277671813965\n",
      "85 4.64390754699707\n",
      "86 4.405582904815674\n",
      "87 4.18079137802124\n",
      "88 3.968458890914917\n",
      "89 3.768059015274048\n",
      "90 3.57924222946167\n",
      "91 3.4009251594543457\n",
      "92 3.232651472091675\n",
      "93 3.0735719203948975\n",
      "94 2.923274040222168\n",
      "95 2.7813515663146973\n",
      "96 2.646982431411743\n",
      "97 2.5198211669921875\n",
      "98 2.3993947505950928\n",
      "99 2.2853763103485107\n",
      "100 2.177335739135742\n",
      "101 2.074908494949341\n",
      "102 1.9777591228485107\n",
      "103 1.8854867219924927\n",
      "104 1.7979234457015991\n",
      "105 1.714813232421875\n",
      "106 1.635840654373169\n",
      "107 1.5608623027801514\n",
      "108 1.4895678758621216\n",
      "109 1.421797513961792\n",
      "110 1.3573706150054932\n",
      "111 1.2960649728775024\n",
      "112 1.2378933429718018\n",
      "113 1.1825839281082153\n",
      "114 1.1299444437026978\n",
      "115 1.0798245668411255\n",
      "116 1.0320863723754883\n",
      "117 0.9865913987159729\n",
      "118 0.9432849884033203\n",
      "119 0.902035117149353\n",
      "120 0.8627598881721497\n",
      "121 0.825330376625061\n",
      "122 0.7896507382392883\n",
      "123 0.7555981874465942\n",
      "124 0.7231500148773193\n",
      "125 0.6922186017036438\n",
      "126 0.6626840829849243\n",
      "127 0.6344841122627258\n",
      "128 0.607585072517395\n",
      "129 0.5819365382194519\n",
      "130 0.5574905872344971\n",
      "131 0.5341120958328247\n",
      "132 0.511806309223175\n",
      "133 0.4905287027359009\n",
      "134 0.4701451361179352\n",
      "135 0.4506863057613373\n",
      "136 0.4320984184741974\n",
      "137 0.41430747509002686\n",
      "138 0.3973069190979004\n",
      "139 0.38103756308555603\n",
      "140 0.3654879033565521\n",
      "141 0.3506124019622803\n",
      "142 0.33637484908103943\n",
      "143 0.32274484634399414\n",
      "144 0.30971312522888184\n",
      "145 0.29722556471824646\n",
      "146 0.28529006242752075\n",
      "147 0.27387529611587524\n",
      "148 0.2629300355911255\n",
      "149 0.25245898962020874\n",
      "150 0.24243243038654327\n",
      "151 0.23282311856746674\n",
      "152 0.22361215949058533\n",
      "153 0.21478471159934998\n",
      "154 0.20633257925510406\n",
      "155 0.1982240080833435\n",
      "156 0.1904599964618683\n",
      "157 0.1830129325389862\n",
      "158 0.17588026821613312\n",
      "159 0.16904529929161072\n",
      "160 0.1624947488307953\n",
      "161 0.15621784329414368\n",
      "162 0.15019989013671875\n",
      "163 0.14441613852977753\n",
      "164 0.1388736516237259\n",
      "165 0.1335511952638626\n",
      "166 0.12844258546829224\n",
      "167 0.12354154139757156\n",
      "168 0.11883655935525894\n",
      "169 0.11431864649057388\n",
      "170 0.10997868329286575\n",
      "171 0.1058143600821495\n",
      "172 0.10181136429309845\n",
      "173 0.0979718416929245\n",
      "174 0.0942811444401741\n",
      "175 0.09073877334594727\n",
      "176 0.08733431994915009\n",
      "177 0.08406402915716171\n",
      "178 0.08092310279607773\n",
      "179 0.07790430635213852\n",
      "180 0.07500220090150833\n",
      "181 0.07221195101737976\n",
      "182 0.06953084468841553\n",
      "183 0.06695376336574554\n",
      "184 0.06447622925043106\n",
      "185 0.06209387630224228\n",
      "186 0.059804368764162064\n",
      "187 0.0576026476919651\n",
      "188 0.05548636615276337\n",
      "189 0.053451359272003174\n",
      "190 0.05149209871888161\n",
      "191 0.049608562141656876\n",
      "192 0.04779217392206192\n",
      "193 0.046045079827308655\n",
      "194 0.044365040957927704\n",
      "195 0.04274820163846016\n",
      "196 0.04119228944182396\n",
      "197 0.03969492018222809\n",
      "198 0.0382549948990345\n",
      "199 0.03687053918838501\n",
      "200 0.03553635627031326\n",
      "201 0.03425237163901329\n",
      "202 0.033017244189977646\n",
      "203 0.03182937204837799\n",
      "204 0.030685797333717346\n",
      "205 0.02958529256284237\n",
      "206 0.028525011613965034\n",
      "207 0.027503199875354767\n",
      "208 0.026520606130361557\n",
      "209 0.02557482197880745\n",
      "210 0.024663565680384636\n",
      "211 0.023786000907421112\n",
      "212 0.022940324619412422\n",
      "213 0.022126292809844017\n",
      "214 0.021342668682336807\n",
      "215 0.02058781497180462\n",
      "216 0.01986049860715866\n",
      "217 0.01915946789085865\n",
      "218 0.018484726548194885\n",
      "219 0.017834331840276718\n",
      "220 0.017207544296979904\n",
      "221 0.016603760421276093\n",
      "222 0.016021955758333206\n",
      "223 0.015460910275578499\n",
      "224 0.014920495450496674\n",
      "225 0.014399759471416473\n",
      "226 0.013897747732698917\n",
      "227 0.013413806445896626\n",
      "228 0.012947270646691322\n",
      "229 0.01249750703573227\n",
      "230 0.012063935399055481\n",
      "231 0.011646388098597527\n",
      "232 0.011243469081819057\n",
      "233 0.010854840278625488\n",
      "234 0.010480293072760105\n",
      "235 0.010119001381099224\n",
      "236 0.009771599434316158\n",
      "237 0.00943632610142231\n",
      "238 0.009113245643675327\n",
      "239 0.008801336400210857\n",
      "240 0.008500536903738976\n",
      "241 0.008210485801100731\n",
      "242 0.00793079100549221\n",
      "243 0.0076606664806604385\n",
      "244 0.007400283589959145\n",
      "245 0.007148953154683113\n",
      "246 0.006906360387802124\n",
      "247 0.006672430317848921\n",
      "248 0.006446641404181719\n",
      "249 0.006228868383914232\n",
      "250 0.006018518470227718\n",
      "251 0.005815559532493353\n",
      "252 0.0056195491924881935\n",
      "253 0.0054304348304867744\n",
      "254 0.0052479589357972145\n",
      "255 0.0050719622522592545\n",
      "256 0.004902026150375605\n",
      "257 0.004738153424113989\n",
      "258 0.004579542204737663\n",
      "259 0.004426536615937948\n",
      "260 0.0042787776328623295\n",
      "261 0.004136137198656797\n",
      "262 0.003998341504484415\n",
      "263 0.003865265753120184\n",
      "264 0.0037367192562669516\n",
      "265 0.0036125830374658108\n",
      "266 0.0034927059896290302\n",
      "267 0.0033769607543945312\n",
      "268 0.0032651741057634354\n",
      "269 0.0031572002917528152\n",
      "270 0.003052984131500125\n",
      "271 0.002952178940176964\n",
      "272 0.002854771912097931\n",
      "273 0.002760813571512699\n",
      "274 0.002669965149834752\n",
      "275 0.002582181477919221\n",
      "276 0.0024974122643470764\n",
      "277 0.0024155930150300264\n",
      "278 0.002336365869268775\n",
      "279 0.0022599082440137863\n",
      "280 0.0021860103588551283\n",
      "281 0.002114602830260992\n",
      "282 0.0020455829799175262\n",
      "283 0.0019788776990026236\n",
      "284 0.0019144138786941767\n",
      "285 0.001852108514867723\n",
      "286 0.0017918958328664303\n",
      "287 0.0017337144818156958\n",
      "288 0.0016774695832282305\n",
      "289 0.001623108983039856\n",
      "290 0.0015705227851867676\n",
      "291 0.001519721350632608\n",
      "292 0.001470606541261077\n",
      "293 0.001423120847903192\n",
      "294 0.0013771740486845374\n",
      "295 0.0013328290078788996\n",
      "296 0.001289909239858389\n",
      "297 0.0012483831960707903\n",
      "298 0.001208233181387186\n",
      "299 0.0011694587301462889\n",
      "300 0.0011319790501147509\n",
      "301 0.001095675746910274\n",
      "302 0.0010605999268591404\n",
      "303 0.0010266214376315475\n",
      "304 0.000993794878013432\n",
      "305 0.0009620738564990461\n",
      "306 0.0009313374175690114\n",
      "307 0.0009016471449285746\n",
      "308 0.0008729249238967896\n",
      "309 0.0008451487519778311\n",
      "310 0.0008182706078514457\n",
      "311 0.0007922449731267989\n",
      "312 0.0007670880295336246\n",
      "313 0.0007427458767779171\n",
      "314 0.0007192314369603992\n",
      "315 0.0006964616477489471\n",
      "316 0.0006744434358552098\n",
      "317 0.0006531089311465621\n",
      "318 0.0006324821733869612\n",
      "319 0.0006125257350504398\n",
      "320 0.0005932265194132924\n",
      "321 0.0005745467497035861\n",
      "322 0.0005564701277762651\n",
      "323 0.0005389846628531814\n",
      "324 0.0005220334278419614\n",
      "325 0.0005056319059804082\n",
      "326 0.0004897785256616771\n",
      "327 0.0004744060570374131\n",
      "328 0.0004595518112182617\n",
      "329 0.0004451785935088992\n",
      "330 0.0004312487435527146\n",
      "331 0.0004177679948043078\n",
      "332 0.00040472886757925153\n",
      "333 0.0003920811868738383\n",
      "334 0.0003798671532422304\n",
      "335 0.00036803592229261994\n",
      "336 0.00035658167325891554\n",
      "337 0.00034549139672890306\n",
      "338 0.00033476162934675813\n",
      "339 0.00032438134076073766\n",
      "340 0.0003143186622764915\n",
      "341 0.00030457042157649994\n",
      "342 0.00029515402275137603\n",
      "343 0.0002860176027752459\n",
      "344 0.00027718491037376225\n",
      "345 0.00026862791855819523\n",
      "346 0.0002603354805614799\n",
      "347 0.00025231344625353813\n",
      "348 0.000244526076130569\n",
      "349 0.00023700404562987387\n",
      "350 0.00022971085854806006\n",
      "351 0.00022264313884079456\n",
      "352 0.00021580375323537737\n",
      "353 0.00020918351947329938\n",
      "354 0.000202763665583916\n",
      "355 0.00019655132200568914\n",
      "356 0.00019052113930229098\n",
      "357 0.00018468461348675191\n",
      "358 0.0001790372043615207\n",
      "359 0.00017356965690851212\n",
      "360 0.00016826746286824346\n",
      "361 0.00016312779916916043\n",
      "362 0.0001581499382155016\n",
      "363 0.00015333380724769086\n",
      "364 0.0001486511464463547\n",
      "365 0.00014412982272915542\n",
      "366 0.0001397471351083368\n",
      "367 0.00013549938739743084\n",
      "368 0.00013138145732227713\n",
      "369 0.00012740024249069393\n",
      "370 0.00012353375495877117\n",
      "371 0.00011978429392911494\n",
      "372 0.00011615628318395466\n",
      "373 0.00011264182830927894\n",
      "374 0.00010923472291324288\n",
      "375 0.00010592213948257267\n",
      "376 0.00010272752115270123\n",
      "377 9.962511830963194e-05\n",
      "378 9.662302181823179e-05\n",
      "379 9.370413317810744e-05\n",
      "380 9.08853835426271e-05\n",
      "381 8.815302862785757e-05\n",
      "382 8.54993486427702e-05\n",
      "383 8.292924758279696e-05\n",
      "384 8.043211710173637e-05\n",
      "385 7.801893661962822e-05\n",
      "386 7.567518332507461e-05\n",
      "387 7.34063214622438e-05\n",
      "388 7.120199006749317e-05\n",
      "389 6.907089846208692e-05\n",
      "390 6.700271478621289e-05\n",
      "391 6.499844312202185e-05\n",
      "392 6.305676652118564e-05\n",
      "393 6.116845179349184e-05\n",
      "394 5.934878208790906e-05\n",
      "395 5.7579498388804495e-05\n",
      "396 5.5859123676782474e-05\n",
      "397 5.419260560302064e-05\n",
      "398 5.257848170003854e-05\n",
      "399 5.1010141760343686e-05\n",
      "400 4.9490710807731375e-05\n",
      "401 4.801948671229184e-05\n",
      "402 4.659402839024551e-05\n",
      "403 4.520896254689433e-05\n",
      "404 4.387163062347099e-05\n",
      "405 4.257502223481424e-05\n",
      "406 4.13043744629249e-05\n",
      "407 4.0081660699797794e-05\n",
      "408 3.889219806296751e-05\n",
      "409 3.7743087887065485e-05\n",
      "410 3.6625533539336175e-05\n",
      "411 3.554358409019187e-05\n",
      "412 3.449434007052332e-05\n",
      "413 3.347314486745745e-05\n",
      "414 3.249033761676401e-05\n",
      "415 3.152757199131884e-05\n",
      "416 3.059856317122467e-05\n",
      "417 2.9695360353798606e-05\n",
      "418 2.8825666959164664e-05\n",
      "419 2.7975689590675756e-05\n",
      "420 2.7151283575221896e-05\n",
      "421 2.6354357032687403e-05\n",
      "422 2.5581010049791075e-05\n",
      "423 2.4827446395647712e-05\n",
      "424 2.4101700546452776e-05\n",
      "425 2.339274578844197e-05\n",
      "426 2.27063792408444e-05\n",
      "427 2.2041753254598007e-05\n",
      "428 2.139542266377248e-05\n",
      "429 2.0769013644894585e-05\n",
      "430 2.016233884205576e-05\n",
      "431 1.957075801328756e-05\n",
      "432 1.9001859982381575e-05\n",
      "433 1.84449709195178e-05\n",
      "434 1.7907304936670698e-05\n",
      "435 1.738423998176586e-05\n",
      "436 1.6880791008588858e-05\n",
      "437 1.6385165508836508e-05\n",
      "438 1.5908699424471706e-05\n",
      "439 1.5446446923306212e-05\n",
      "440 1.4995764104241971e-05\n",
      "441 1.4561704119842034e-05\n",
      "442 1.4136417121335398e-05\n",
      "443 1.372732913296204e-05\n",
      "444 1.3329673492989969e-05\n",
      "445 1.2941608474648092e-05\n",
      "446 1.2565846191137098e-05\n",
      "447 1.2201953722978942e-05\n",
      "448 1.184782558993902e-05\n",
      "449 1.1502427696541417e-05\n",
      "450 1.1168946912221145e-05\n",
      "451 1.0846071745618246e-05\n",
      "452 1.0531219231779687e-05\n",
      "453 1.0226431186310947e-05\n",
      "454 9.931209206115454e-06\n",
      "455 9.646510079619475e-06\n",
      "456 9.367839993501548e-06\n",
      "457 9.096088433580007e-06\n",
      "458 8.83518259797711e-06\n",
      "459 8.580371286370791e-06\n",
      "460 8.332269317179453e-06\n",
      "461 8.092404641502071e-06\n",
      "462 7.85930660640588e-06\n",
      "463 7.634341272932943e-06\n",
      "464 7.414140327455243e-06\n",
      "465 7.199668289104011e-06\n",
      "466 6.991305781411938e-06\n",
      "467 6.791764917579712e-06\n",
      "468 6.597300853172783e-06\n",
      "469 6.407233286154224e-06\n",
      "470 6.224850949365646e-06\n",
      "471 6.0454290178313386e-06\n",
      "472 5.871434950677212e-06\n",
      "473 5.704107024939731e-06\n",
      "474 5.5395457820850424e-06\n",
      "475 5.382243671192555e-06\n",
      "476 5.227356268733274e-06\n",
      "477 5.077775767858839e-06\n",
      "478 4.931834155286197e-06\n",
      "479 4.790485490957508e-06\n",
      "480 4.653531959775137e-06\n",
      "481 4.5201677494333126e-06\n",
      "482 4.390817139210412e-06\n",
      "483 4.266916221240535e-06\n",
      "484 4.14420128436177e-06\n",
      "485 4.027348950330634e-06\n",
      "486 3.911978183168685e-06\n",
      "487 3.800290414801566e-06\n",
      "488 3.6919502690579975e-06\n",
      "489 3.5875664252671413e-06\n",
      "490 3.4860991036111955e-06\n",
      "491 3.3865430850710254e-06\n",
      "492 3.2901855320233153e-06\n",
      "493 3.197465048287995e-06\n",
      "494 3.106325038970681e-06\n",
      "495 3.018062670889776e-06\n",
      "496 2.9327557058422826e-06\n",
      "497 2.848682925105095e-06\n",
      "498 2.767805881376262e-06\n",
      "499 2.6896491363004316e-06\n"
     ]
    }
   ],
   "source": [
    "# 可运行代码见本文件夹中的 two_layer_net_nn.py\n",
    "import torch\n",
    "\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "# N是批大小；D是输入维度\n",
    "# H是隐藏层维度；D_out是输出维度\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 产生输入和输出随机张量\n",
    "x = torch.randn(N, D_in, device=device)\n",
    "y = torch.randn(N, D_out, device=device)\n",
    "\n",
    "\n",
    "# 使用nn包将我们的模型定义为一系列的层。\n",
    "# nn.Sequential是包含其他模块的模块，并按顺序应用这些模块来产生其输出。\n",
    "# 每个线性模块使用线性函数从输入计算输出，并保存其内部的权重和偏差张量。\n",
    "# 在构造模型之后，我们使用.to()方法将其移动到所需的设备。\n",
    "model = torch.nn.Sequential(\n",
    "            torch.nn.Linear(D_in, H),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(H, D_out),\n",
    "        ).to(device)\n",
    "\n",
    "\n",
    "# nn包还包含常用的损失函数的定义；\n",
    "# 在这种情况下，我们将使用平均平方误差(MSE)作为我们的损失函数。\n",
    "# 设置reduction='sum'，表示我们计算的是平方误差的“和”，而不是平均值;\n",
    "# 这是为了与前面我们手工计算损失的例子保持一致，\n",
    "# 但是在实践中，通过设置reduction='elementwise_mean'来使用均方误差作为损失更为常见。\n",
    "loss_fn = torch.nn.MSELoss(reduction='sum')\n",
    "\n",
    "learning_rate = 1e-4\n",
    "for t in range(500):\n",
    "\n",
    "    # 前向传播：通过向模型传入x计算预测的y。\n",
    "    # 模块对象重载了__call__运算符，所以可以像函数那样调用它们。\n",
    "    # 这么做相当于向模块传入了一个张量，然后它返回了一个输出张量。\n",
    "    y_pred = model(x)\n",
    "\n",
    "    # 计算并打印损失。我们传递包含y的预测值和真实值的张量，损失函数返回包含损失的张量。\n",
    "    loss = loss_fn(y_pred, y)\n",
    "    print(t, loss.item())\n",
    "\n",
    "    # 反向传播之前清零梯度\n",
    "    model.zero_grad()\n",
    "\n",
    "    # 反向传播：计算模型的损失对所有可学习参数的导数(梯度）。\n",
    "    # 在内部，每个模块的参数存储在requires_grad=True的张量中，\n",
    "    # 因此这个调用将计算模型中所有可学习参数的梯度。\n",
    "    loss.backward()\n",
    "\n",
    "    # 使用梯度下降更新权重。\n",
    "    # 每个参数都是张量，所以我们可以像我们以前那样可以得到它的数值和梯度\n",
    "    with torch.no_grad():\n",
    "        for param in model.parameters():\n",
    "            param.data -= learning_rate * param.grad\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 708.49755859375\n",
      "1 690.7189331054688\n",
      "2 673.5602416992188\n",
      "3 656.8994140625\n",
      "4 640.7457885742188\n",
      "5 625.2139892578125\n",
      "6 610.0953369140625\n",
      "7 595.3345336914062\n",
      "8 580.9495849609375\n",
      "9 566.8880615234375\n",
      "10 553.1664428710938\n",
      "11 539.8731689453125\n",
      "12 526.9081420898438\n",
      "13 514.2791748046875\n",
      "14 502.00494384765625\n",
      "15 490.1575927734375\n",
      "16 478.6865234375\n",
      "17 467.47210693359375\n",
      "18 456.5691223144531\n",
      "19 445.9284362792969\n",
      "20 435.5552978515625\n",
      "21 425.47320556640625\n",
      "22 415.6747741699219\n",
      "23 406.1044006347656\n",
      "24 396.78057861328125\n",
      "25 387.67706298828125\n",
      "26 378.7705993652344\n",
      "27 370.09478759765625\n",
      "28 361.724609375\n",
      "29 353.6076354980469\n",
      "30 345.691650390625\n",
      "31 337.9635925292969\n",
      "32 330.3829040527344\n",
      "33 322.97802734375\n",
      "34 315.7098083496094\n",
      "35 308.6239013671875\n",
      "36 301.7070617675781\n",
      "37 294.9850769042969\n",
      "38 288.4387512207031\n",
      "39 282.0312194824219\n",
      "40 275.76513671875\n",
      "41 269.6101989746094\n",
      "42 263.5793762207031\n",
      "43 257.6697082519531\n",
      "44 251.88914489746094\n",
      "45 246.21063232421875\n",
      "46 240.64321899414062\n",
      "47 235.18353271484375\n",
      "48 229.837158203125\n",
      "49 224.6078338623047\n",
      "50 219.47760009765625\n",
      "51 214.46896362304688\n",
      "52 209.56588745117188\n",
      "53 204.76715087890625\n",
      "54 200.0802001953125\n",
      "55 195.47366333007812\n",
      "56 190.94918823242188\n",
      "57 186.51458740234375\n",
      "58 182.17323303222656\n",
      "59 177.9049530029297\n",
      "60 173.71986389160156\n",
      "61 169.62197875976562\n",
      "62 165.5968475341797\n",
      "63 161.64866638183594\n",
      "64 157.77114868164062\n",
      "65 153.9698944091797\n",
      "66 150.23944091796875\n",
      "67 146.58489990234375\n",
      "68 143.0028076171875\n",
      "69 139.48834228515625\n",
      "70 136.04551696777344\n",
      "71 132.67010498046875\n",
      "72 129.35687255859375\n",
      "73 126.11380767822266\n",
      "74 122.93473052978516\n",
      "75 119.82479858398438\n",
      "76 116.78187561035156\n",
      "77 113.79852294921875\n",
      "78 110.87940216064453\n",
      "79 108.02103424072266\n",
      "80 105.22433471679688\n",
      "81 102.48342895507812\n",
      "82 99.79974365234375\n",
      "83 97.17254638671875\n",
      "84 94.60012817382812\n",
      "85 92.07320404052734\n",
      "86 89.59675598144531\n",
      "87 87.17415618896484\n",
      "88 84.80549621582031\n",
      "89 82.4862289428711\n",
      "90 80.21369934082031\n",
      "91 77.98887634277344\n",
      "92 75.80968475341797\n",
      "93 73.68115234375\n",
      "94 71.59922790527344\n",
      "95 69.56292724609375\n",
      "96 67.56936645507812\n",
      "97 65.62421417236328\n",
      "98 63.72151184082031\n",
      "99 61.861358642578125\n",
      "100 60.04894256591797\n",
      "101 58.27788543701172\n",
      "102 56.54726791381836\n",
      "103 54.85771179199219\n",
      "104 53.20332717895508\n",
      "105 51.590232849121094\n",
      "106 50.01502990722656\n",
      "107 48.47751235961914\n",
      "108 46.977081298828125\n",
      "109 45.51768493652344\n",
      "110 44.09311294555664\n",
      "111 42.70606994628906\n",
      "112 41.35687255859375\n",
      "113 40.04254150390625\n",
      "114 38.76177215576172\n",
      "115 37.51500701904297\n",
      "116 36.303401947021484\n",
      "117 35.122772216796875\n",
      "118 33.97451400756836\n",
      "119 32.85887908935547\n",
      "120 31.772083282470703\n",
      "121 30.717159271240234\n",
      "122 29.692110061645508\n",
      "123 28.69643783569336\n",
      "124 27.729297637939453\n",
      "125 26.791723251342773\n",
      "126 25.882720947265625\n",
      "127 25.000804901123047\n",
      "128 24.14533233642578\n",
      "129 23.316822052001953\n",
      "130 22.51276969909668\n",
      "131 21.735095977783203\n",
      "132 20.981468200683594\n",
      "133 20.253068923950195\n",
      "134 19.54608917236328\n",
      "135 18.86163330078125\n",
      "136 18.199573516845703\n",
      "137 17.55841827392578\n",
      "138 16.93792724609375\n",
      "139 16.337663650512695\n",
      "140 15.756808280944824\n",
      "141 15.19473934173584\n",
      "142 14.651498794555664\n",
      "143 14.126349449157715\n",
      "144 13.619394302368164\n",
      "145 13.129375457763672\n",
      "146 12.655590057373047\n",
      "147 12.197501182556152\n",
      "148 11.75515365600586\n",
      "149 11.32832145690918\n",
      "150 10.915675163269043\n",
      "151 10.517701148986816\n",
      "152 10.133268356323242\n",
      "153 9.761712074279785\n",
      "154 9.40346622467041\n",
      "155 9.057785034179688\n",
      "156 8.724125862121582\n",
      "157 8.402630805969238\n",
      "158 8.092414855957031\n",
      "159 7.7930121421813965\n",
      "160 7.504303455352783\n",
      "161 7.225708961486816\n",
      "162 6.9573259353637695\n",
      "163 6.698764801025391\n",
      "164 6.450048446655273\n",
      "165 6.210200309753418\n",
      "166 5.979354381561279\n",
      "167 5.756897449493408\n",
      "168 5.543187618255615\n",
      "169 5.337346076965332\n",
      "170 5.138973236083984\n",
      "171 4.948029518127441\n",
      "172 4.764311790466309\n",
      "173 4.587728977203369\n",
      "174 4.417765140533447\n",
      "175 4.2540364265441895\n",
      "176 4.096462249755859\n",
      "177 3.9450833797454834\n",
      "178 3.799302101135254\n",
      "179 3.659001588821411\n",
      "180 3.5239853858947754\n",
      "181 3.3940906524658203\n",
      "182 3.2688987255096436\n",
      "183 3.1484367847442627\n",
      "184 3.032536506652832\n",
      "185 2.921036958694458\n",
      "186 2.8136117458343506\n",
      "187 2.7102675437927246\n",
      "188 2.610814094543457\n",
      "189 2.51505184173584\n",
      "190 2.4228527545928955\n",
      "191 2.334129810333252\n",
      "192 2.2487752437591553\n",
      "193 2.166490077972412\n",
      "194 2.087411880493164\n",
      "195 2.0112357139587402\n",
      "196 1.9379332065582275\n",
      "197 1.8674243688583374\n",
      "198 1.7994303703308105\n",
      "199 1.7340221405029297\n",
      "200 1.6710797548294067\n",
      "201 1.6104768514633179\n",
      "202 1.5521494150161743\n",
      "203 1.4959542751312256\n",
      "204 1.4418638944625854\n",
      "205 1.3897416591644287\n",
      "206 1.3395805358886719\n",
      "207 1.291266679763794\n",
      "208 1.24467933177948\n",
      "209 1.1998751163482666\n",
      "210 1.1566497087478638\n",
      "211 1.1150565147399902\n",
      "212 1.0750007629394531\n",
      "213 1.0363986492156982\n",
      "214 0.9992039203643799\n",
      "215 0.9633617997169495\n",
      "216 0.9288045763969421\n",
      "217 0.8955651521682739\n",
      "218 0.8635635375976562\n",
      "219 0.8327074646949768\n",
      "220 0.8029590845108032\n",
      "221 0.7742791771888733\n",
      "222 0.746661901473999\n",
      "223 0.7200335264205933\n",
      "224 0.6943544149398804\n",
      "225 0.6696036458015442\n",
      "226 0.6457059979438782\n",
      "227 0.6226829290390015\n",
      "228 0.6004749536514282\n",
      "229 0.579059362411499\n",
      "230 0.5583828687667847\n",
      "231 0.5384621620178223\n",
      "232 0.5192241072654724\n",
      "233 0.5006882548332214\n",
      "234 0.4827725887298584\n",
      "235 0.4655084013938904\n",
      "236 0.4488493800163269\n",
      "237 0.4327702820301056\n",
      "238 0.41726377606391907\n",
      "239 0.40229251980781555\n",
      "240 0.38785290718078613\n",
      "241 0.37391170859336853\n",
      "242 0.36045992374420166\n",
      "243 0.34748148918151855\n",
      "244 0.33495667576789856\n",
      "245 0.3228825330734253\n",
      "246 0.31121718883514404\n",
      "247 0.29994890093803406\n",
      "248 0.2890838086605072\n",
      "249 0.27859923243522644\n",
      "250 0.2684836983680725\n",
      "251 0.25871726870536804\n",
      "252 0.2492939531803131\n",
      "253 0.24019990861415863\n",
      "254 0.2314232438802719\n",
      "255 0.2229534536600113\n",
      "256 0.21478237211704254\n",
      "257 0.20689770579338074\n",
      "258 0.19928789138793945\n",
      "259 0.19194358587265015\n",
      "260 0.18485945463180542\n",
      "261 0.17802411317825317\n",
      "262 0.17142948508262634\n",
      "263 0.16507016122341156\n",
      "264 0.1589289903640747\n",
      "265 0.15300679206848145\n",
      "266 0.14729569852352142\n",
      "267 0.1417878270149231\n",
      "268 0.13649962842464447\n",
      "269 0.13140670955181122\n",
      "270 0.12648779153823853\n",
      "271 0.12174779176712036\n",
      "272 0.11718187481164932\n",
      "273 0.11277642101049423\n",
      "274 0.10852889716625214\n",
      "275 0.10443525016307831\n",
      "276 0.1004875972867012\n",
      "277 0.09668319672346115\n",
      "278 0.09301703423261642\n",
      "279 0.08948283642530441\n",
      "280 0.08607643842697144\n",
      "281 0.08279428631067276\n",
      "282 0.07963074743747711\n",
      "283 0.07658359408378601\n",
      "284 0.07364695519208908\n",
      "285 0.07081808894872665\n",
      "286 0.06809262931346893\n",
      "287 0.06546705216169357\n",
      "288 0.06293834000825882\n",
      "289 0.06050319969654083\n",
      "290 0.058157917112112045\n",
      "291 0.05589763820171356\n",
      "292 0.05372346192598343\n",
      "293 0.05162664130330086\n",
      "294 0.049609165638685226\n",
      "295 0.047667261213064194\n",
      "296 0.0457976870238781\n",
      "297 0.04399765282869339\n",
      "298 0.042265456169843674\n",
      "299 0.04059785604476929\n",
      "300 0.03899315744638443\n",
      "301 0.0374491848051548\n",
      "302 0.03596466779708862\n",
      "303 0.0345342680811882\n",
      "304 0.03316863626241684\n",
      "305 0.03187761828303337\n",
      "306 0.03063683770596981\n",
      "307 0.029444461688399315\n",
      "308 0.028298331424593925\n",
      "309 0.02719690278172493\n",
      "310 0.026138100773096085\n",
      "311 0.025120418518781662\n",
      "312 0.024142345413565636\n",
      "313 0.023202845826745033\n",
      "314 0.022299738600850105\n",
      "315 0.021431615576148033\n",
      "316 0.020597675815224648\n",
      "317 0.01979612372815609\n",
      "318 0.01902562566101551\n",
      "319 0.018284952268004417\n",
      "320 0.017572833225131035\n",
      "321 0.016888191923499107\n",
      "322 0.016230396926403046\n",
      "323 0.01559798140078783\n",
      "324 0.01499020867049694\n",
      "325 0.014405664056539536\n",
      "326 0.013843666762113571\n",
      "327 0.013303529471158981\n",
      "328 0.012784171849489212\n",
      "329 0.012284836731851101\n",
      "330 0.011804649606347084\n",
      "331 0.011343056336045265\n",
      "332 0.010899143293499947\n",
      "333 0.010472366586327553\n",
      "334 0.010062016546726227\n",
      "335 0.009667606092989445\n",
      "336 0.009288441389799118\n",
      "337 0.008923922665417194\n",
      "338 0.00857336726039648\n",
      "339 0.008236224763095379\n",
      "340 0.007912170141935349\n",
      "341 0.007600465789437294\n",
      "342 0.007300953846424818\n",
      "343 0.007012917660176754\n",
      "344 0.006735962815582752\n",
      "345 0.006469876505434513\n",
      "346 0.006214039400219917\n",
      "347 0.0059682028368115425\n",
      "348 0.005731713958084583\n",
      "349 0.0055044712498784065\n",
      "350 0.005286111030727625\n",
      "351 0.0050762249156832695\n",
      "352 0.004874602425843477\n",
      "353 0.004680761136114597\n",
      "354 0.004494529217481613\n",
      "355 0.00431553740054369\n",
      "356 0.0041435277089476585\n",
      "357 0.003978306427598\n",
      "358 0.003819461213424802\n",
      "359 0.003666945733129978\n",
      "360 0.0035204372834414244\n",
      "361 0.0033796578645706177\n",
      "362 0.0032442996744066477\n",
      "363 0.0031143263913691044\n",
      "364 0.0029894921462982893\n",
      "365 0.002869540359824896\n",
      "366 0.0027543886099010706\n",
      "367 0.0026437193155288696\n",
      "368 0.0025374083779752254\n",
      "369 0.002435301896184683\n",
      "370 0.002337238285690546\n",
      "371 0.0022429984528571367\n",
      "372 0.002152535365894437\n",
      "373 0.0020657270215451717\n",
      "374 0.0019823075272142887\n",
      "375 0.0019021080806851387\n",
      "376 0.0018250967841595411\n",
      "377 0.0017512195045128465\n",
      "378 0.001680301851592958\n",
      "379 0.0016121517401188612\n",
      "380 0.0015466820914298296\n",
      "381 0.001483852625824511\n",
      "382 0.0014235122362151742\n",
      "383 0.001365650212392211\n",
      "384 0.001310022664256394\n",
      "385 0.0012566718505695462\n",
      "386 0.0012053991667926311\n",
      "387 0.0011561890132725239\n",
      "388 0.0011089653708040714\n",
      "389 0.001063615083694458\n",
      "390 0.0010201449040323496\n",
      "391 0.0009783412097021937\n",
      "392 0.0009382619173265994\n",
      "393 0.0008997655240818858\n",
      "394 0.0008628303767181933\n",
      "395 0.0008273747516795993\n",
      "396 0.0007933661690913141\n",
      "397 0.000760699505917728\n",
      "398 0.0007293742382898927\n",
      "399 0.0006992922862991691\n",
      "400 0.0006704454426653683\n",
      "401 0.0006427380722016096\n",
      "402 0.000616166100371629\n",
      "403 0.0005906709120608866\n",
      "404 0.0005661866744048893\n",
      "405 0.0005427382420748472\n",
      "406 0.0005202167667448521\n",
      "407 0.000498609384521842\n",
      "408 0.0004778979637194425\n",
      "409 0.0004579942615237087\n",
      "410 0.0004389273817650974\n",
      "411 0.0004206090816296637\n",
      "412 0.000403072772314772\n",
      "413 0.00038622686406597495\n",
      "414 0.0003700896049849689\n",
      "415 0.0003546103253029287\n",
      "416 0.0003397362306714058\n",
      "417 0.0003255030023865402\n",
      "418 0.000311838899506256\n",
      "419 0.0002987319603562355\n",
      "420 0.00028617557836696506\n",
      "421 0.00027412374038249254\n",
      "422 0.0002625775523483753\n",
      "423 0.0002515054075047374\n",
      "424 0.00024087633937597275\n",
      "425 0.00023070545285008848\n",
      "426 0.00022094872838351876\n",
      "427 0.00021158918389119208\n",
      "428 0.00020262025645934045\n",
      "429 0.0001940196380019188\n",
      "430 0.00018578875460661948\n",
      "431 0.00017787227989174426\n",
      "432 0.0001703126181382686\n",
      "433 0.00016305041208397597\n",
      "434 0.00015609327238053083\n",
      "435 0.0001494266907684505\n",
      "436 0.00014303326315712184\n",
      "437 0.00013690703781321645\n",
      "438 0.0001310365623794496\n",
      "439 0.00012542359763756394\n",
      "440 0.00012003118899883702\n",
      "441 0.00011487712617963552\n",
      "442 0.00010992810712195933\n",
      "443 0.00010518521594349295\n",
      "444 0.00010065671085612848\n",
      "445 9.630822023609653e-05\n",
      "446 9.21469327295199e-05\n",
      "447 8.815184992272407e-05\n",
      "448 8.433105540461838e-05\n",
      "449 8.067619637586176e-05\n",
      "450 7.716863910900429e-05\n",
      "451 7.381564500974491e-05\n",
      "452 7.059812924126163e-05\n",
      "453 6.751693581463769e-05\n",
      "454 6.457241397583857e-05\n",
      "455 6.174424197524786e-05\n",
      "456 5.9049645642517135e-05\n",
      "457 5.645746205118485e-05\n",
      "458 5.398214125307277e-05\n",
      "459 5.161449371371418e-05\n",
      "460 4.934116805088706e-05\n",
      "461 4.716833063866943e-05\n",
      "462 4.509007339947857e-05\n",
      "463 4.30994296038989e-05\n",
      "464 4.119313962291926e-05\n",
      "465 3.9369107980746776e-05\n",
      "466 3.762839696719311e-05\n",
      "467 3.595451926230453e-05\n",
      "468 3.436473343754187e-05\n",
      "469 3.283427577116527e-05\n",
      "470 3.137251042062417e-05\n",
      "471 2.9973884011269547e-05\n",
      "472 2.8633348847506568e-05\n",
      "473 2.7352692995918915e-05\n",
      "474 2.613159995235037e-05\n",
      "475 2.4956341803772375e-05\n",
      "476 2.383425089647062e-05\n",
      "477 2.2763309971196577e-05\n",
      "478 2.173848042730242e-05\n",
      "479 2.0760508050443605e-05\n",
      "480 1.9822142348857597e-05\n",
      "481 1.8931656086351722e-05\n",
      "482 1.8072338207275607e-05\n",
      "483 1.725084803183563e-05\n",
      "484 1.646862983761821e-05\n",
      "485 1.5720199371571653e-05\n",
      "486 1.5005789464339614e-05\n",
      "487 1.432005774404388e-05\n",
      "488 1.3666938684764318e-05\n",
      "489 1.30434409584268e-05\n",
      "490 1.2445498214219697e-05\n",
      "491 1.1873902622028254e-05\n",
      "492 1.133276782638859e-05\n",
      "493 1.0812169421114959e-05\n",
      "494 1.0312690392311197e-05\n",
      "495 9.836990102485288e-06\n",
      "496 9.380878509546164e-06\n",
      "497 8.949135917646345e-06\n",
      "498 8.53522215038538e-06\n",
      "499 8.137144504871685e-06\n"
     ]
    }
   ],
   "source": [
    "# 可运行代码见本文件夹中的 two_layer_net_optim.py\n",
    "import torch\n",
    "\n",
    "# N是批大小；D是输入维度\n",
    "# H是隐藏层维度；D_out是输出维度\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 产生随机输入和输出张量\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "# 使用nn包定义模型和损失函数\n",
    "model = torch.nn.Sequential(\n",
    "          torch.nn.Linear(D_in, H),\n",
    "          torch.nn.ReLU(),\n",
    "          torch.nn.Linear(H, D_out),\n",
    "        )\n",
    "loss_fn = torch.nn.MSELoss(reduction='sum')\n",
    "\n",
    "# 使用optim包定义优化器(Optimizer）。Optimizer将会为我们更新模型的权重。\n",
    "# 这里我们使用Adam优化方法；optim包还包含了许多别的优化算法。\n",
    "# Adam构造函数的第一个参数告诉优化器应该更新哪些张量。\n",
    "learning_rate = 1e-4\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "for t in range(500):\n",
    "\n",
    "    # 前向传播：通过像模型输入x计算预测的y\n",
    "    y_pred = model(x)\n",
    "\n",
    "    # 计算并打印loss\n",
    "    loss = loss_fn(y_pred, y)\n",
    "    print(t, loss.item())\n",
    "\n",
    "    # 在反向传播之前，使用optimizer将它要更新的所有张量的梯度清零(这些张量是模型可学习的权重)\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    # 反向传播：根据模型的参数计算loss的梯度\n",
    "    loss.backward()\n",
    "\n",
    "    # 调用Optimizer的step函数使它所有参数更新\n",
    "    optimizer.step()\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 670.3330078125\n",
      "1 622.1351318359375\n",
      "2 580.6025390625\n",
      "3 544.3514404296875\n",
      "4 512.1488037109375\n",
      "5 482.88214111328125\n",
      "6 456.1648254394531\n",
      "7 431.47186279296875\n",
      "8 408.5321960449219\n",
      "9 387.19830322265625\n",
      "10 367.2596435546875\n",
      "11 348.5539245605469\n",
      "12 330.8318176269531\n",
      "13 313.9499206542969\n",
      "14 297.8343505859375\n",
      "15 282.4521484375\n",
      "16 267.8585205078125\n",
      "17 253.95721435546875\n",
      "18 240.6961669921875\n",
      "19 228.0515899658203\n",
      "20 215.97691345214844\n",
      "21 204.40687561035156\n",
      "22 193.38223266601562\n",
      "23 182.81039428710938\n",
      "24 172.72833251953125\n",
      "25 163.1316375732422\n",
      "26 153.98129272460938\n",
      "27 145.28538513183594\n",
      "28 137.02255249023438\n",
      "29 129.1773681640625\n",
      "30 121.7394027709961\n",
      "31 114.69378662109375\n",
      "32 108.0101547241211\n",
      "33 101.68169403076172\n",
      "34 95.70033264160156\n",
      "35 90.03951263427734\n",
      "36 84.68370056152344\n",
      "37 79.61115264892578\n",
      "38 74.81372833251953\n",
      "39 70.26966857910156\n",
      "40 65.99078369140625\n",
      "41 61.964447021484375\n",
      "42 58.17949676513672\n",
      "43 54.62434387207031\n",
      "44 51.28446960449219\n",
      "45 48.13945007324219\n",
      "46 45.19004821777344\n",
      "47 42.4260368347168\n",
      "48 39.838340759277344\n",
      "49 37.40973663330078\n",
      "50 35.13330841064453\n",
      "51 33.004005432128906\n",
      "52 31.010618209838867\n",
      "53 29.144962310791016\n",
      "54 27.399457931518555\n",
      "55 25.766277313232422\n",
      "56 24.23766326904297\n",
      "57 22.808502197265625\n",
      "58 21.470561981201172\n",
      "59 20.21601104736328\n",
      "60 19.04170799255371\n",
      "61 17.943408966064453\n",
      "62 16.915063858032227\n",
      "63 15.952703475952148\n",
      "64 15.050783157348633\n",
      "65 14.204870223999023\n",
      "66 13.411467552185059\n",
      "67 12.667774200439453\n",
      "68 11.970329284667969\n",
      "69 11.315000534057617\n",
      "70 10.699311256408691\n",
      "71 10.121505737304688\n",
      "72 9.57691764831543\n",
      "73 9.065089225769043\n",
      "74 8.583870887756348\n",
      "75 8.131742477416992\n",
      "76 7.706265926361084\n",
      "77 7.306112289428711\n",
      "78 6.929203033447266\n",
      "79 6.574184417724609\n",
      "80 6.239349842071533\n",
      "81 5.923886299133301\n",
      "82 5.62738037109375\n",
      "83 5.348021984100342\n",
      "84 5.084000587463379\n",
      "85 4.834926605224609\n",
      "86 4.599518299102783\n",
      "87 4.377081394195557\n",
      "88 4.166012763977051\n",
      "89 3.9669582843780518\n",
      "90 3.7786102294921875\n",
      "91 3.6005806922912598\n",
      "92 3.432103395462036\n",
      "93 3.272714138031006\n",
      "94 3.121401071548462\n",
      "95 2.977987051010132\n",
      "96 2.8416237831115723\n",
      "97 2.7122161388397217\n",
      "98 2.589386463165283\n",
      "99 2.4727282524108887\n",
      "100 2.3620049953460693\n",
      "101 2.2568600177764893\n",
      "102 2.1570606231689453\n",
      "103 2.062227249145508\n",
      "104 1.9719268083572388\n",
      "105 1.8860158920288086\n",
      "106 1.8043349981307983\n",
      "107 1.7265948057174683\n",
      "108 1.6525304317474365\n",
      "109 1.5820380449295044\n",
      "110 1.5149924755096436\n",
      "111 1.4509773254394531\n",
      "112 1.38998281955719\n",
      "113 1.3319194316864014\n",
      "114 1.2765388488769531\n",
      "115 1.223703384399414\n",
      "116 1.1732513904571533\n",
      "117 1.1251002550125122\n",
      "118 1.0791445970535278\n",
      "119 1.0352662801742554\n",
      "120 0.993349015712738\n",
      "121 0.9532530307769775\n",
      "122 0.9149729013442993\n",
      "123 0.8783961534500122\n",
      "124 0.8433940410614014\n",
      "125 0.8098951578140259\n",
      "126 0.7778967618942261\n",
      "127 0.7472296953201294\n",
      "128 0.7178683280944824\n",
      "129 0.6897790431976318\n",
      "130 0.6629368662834167\n",
      "131 0.6372953653335571\n",
      "132 0.6127249598503113\n",
      "133 0.5891867280006409\n",
      "134 0.5666249990463257\n",
      "135 0.5449991226196289\n",
      "136 0.5242858529090881\n",
      "137 0.5045913457870483\n",
      "138 0.4857090711593628\n",
      "139 0.4675922095775604\n",
      "140 0.4502110481262207\n",
      "141 0.4335486590862274\n",
      "142 0.4175534248352051\n",
      "143 0.40219584107398987\n",
      "144 0.38743752241134644\n",
      "145 0.37327995896339417\n",
      "146 0.35967469215393066\n",
      "147 0.3466087281703949\n",
      "148 0.3340657949447632\n",
      "149 0.3220197856426239\n",
      "150 0.31044429540634155\n",
      "151 0.29931920766830444\n",
      "152 0.2886319160461426\n",
      "153 0.27834469079971313\n",
      "154 0.26846423745155334\n",
      "155 0.25897032022476196\n",
      "156 0.2498343288898468\n",
      "157 0.24104544520378113\n",
      "158 0.23258957266807556\n",
      "159 0.2244509756565094\n",
      "160 0.21661967039108276\n",
      "161 0.20908048748970032\n",
      "162 0.20181997120380402\n",
      "163 0.19483217597007751\n",
      "164 0.18809771537780762\n",
      "165 0.18161088228225708\n",
      "166 0.17536482214927673\n",
      "167 0.16934451460838318\n",
      "168 0.16354802250862122\n",
      "169 0.15795984864234924\n",
      "170 0.15257468819618225\n",
      "171 0.14738447964191437\n",
      "172 0.14238455891609192\n",
      "173 0.1375609189271927\n",
      "174 0.13291282951831818\n",
      "175 0.12843424081802368\n",
      "176 0.12411428987979889\n",
      "177 0.11994710564613342\n",
      "178 0.11592580378055573\n",
      "179 0.11205082386732101\n",
      "180 0.10831494629383087\n",
      "181 0.10470694303512573\n",
      "182 0.10122723132371902\n",
      "183 0.09787022322416306\n",
      "184 0.09463103860616684\n",
      "185 0.09150409698486328\n",
      "186 0.08848537504673004\n",
      "187 0.08557063341140747\n",
      "188 0.08275772631168365\n",
      "189 0.0800425261259079\n",
      "190 0.07742227613925934\n",
      "191 0.07489611953496933\n",
      "192 0.07245497405529022\n",
      "193 0.07009787112474442\n",
      "194 0.06782012432813644\n",
      "195 0.0656198114156723\n",
      "196 0.06349606812000275\n",
      "197 0.06144504249095917\n",
      "198 0.059464942663908005\n",
      "199 0.05755111947655678\n",
      "200 0.05570167675614357\n",
      "201 0.05391577258706093\n",
      "202 0.05218932777643204\n",
      "203 0.05051891505718231\n",
      "204 0.048904892057180405\n",
      "205 0.04734548553824425\n",
      "206 0.045836951583623886\n",
      "207 0.04437888786196709\n",
      "208 0.042969267815351486\n",
      "209 0.0416058711707592\n",
      "210 0.0402880497276783\n",
      "211 0.03901461511850357\n",
      "212 0.03778231889009476\n",
      "213 0.03658979758620262\n",
      "214 0.035435885190963745\n",
      "215 0.034319959580898285\n",
      "216 0.033240906894207\n",
      "217 0.03219662606716156\n",
      "218 0.031186804175376892\n",
      "219 0.030210351571440697\n",
      "220 0.02926485426723957\n",
      "221 0.028350071981549263\n",
      "222 0.027465030550956726\n",
      "223 0.026608699932694435\n",
      "224 0.02577926404774189\n",
      "225 0.024976860731840134\n",
      "226 0.02420009672641754\n",
      "227 0.023448016494512558\n",
      "228 0.0227208249270916\n",
      "229 0.022017527371644974\n",
      "230 0.021336225792765617\n",
      "231 0.020676400512456894\n",
      "232 0.020037813112139702\n",
      "233 0.01941952109336853\n",
      "234 0.018820926547050476\n",
      "235 0.018241040408611298\n",
      "236 0.01767946220934391\n",
      "237 0.017135633155703545\n",
      "238 0.016609130427241325\n",
      "239 0.016100136563181877\n",
      "240 0.015607097186148167\n",
      "241 0.015129368752241135\n",
      "242 0.014667052775621414\n",
      "243 0.014218839816749096\n",
      "244 0.013784943148493767\n",
      "245 0.013364620506763458\n",
      "246 0.012957354076206684\n",
      "247 0.01256288681179285\n",
      "248 0.012180707417428493\n",
      "249 0.011810306459665298\n",
      "250 0.011451484635472298\n",
      "251 0.011103850789368153\n",
      "252 0.010767027735710144\n",
      "253 0.010440577752888203\n",
      "254 0.010124352760612965\n",
      "255 0.009818021208047867\n",
      "256 0.009521061554551125\n",
      "257 0.009233087301254272\n",
      "258 0.008954175747931004\n",
      "259 0.008683871477842331\n",
      "260 0.008421863429248333\n",
      "261 0.008167926222085953\n",
      "262 0.00792172085493803\n",
      "263 0.007683159783482552\n",
      "264 0.00745187746360898\n",
      "265 0.007227757945656776\n",
      "266 0.007010575849562883\n",
      "267 0.006799966096878052\n",
      "268 0.006595676764845848\n",
      "269 0.006397766526788473\n",
      "270 0.0062059685587882996\n",
      "271 0.006019872147589922\n",
      "272 0.005839570425450802\n",
      "273 0.005664771888405085\n",
      "274 0.005495148710906506\n",
      "275 0.005330785643309355\n",
      "276 0.005171436816453934\n",
      "277 0.0050168572925031185\n",
      "278 0.004866957664489746\n",
      "279 0.004721666686236858\n",
      "280 0.004580778069794178\n",
      "281 0.004444214049726725\n",
      "282 0.004311767872422934\n",
      "283 0.004183291923254728\n",
      "284 0.00405867351219058\n",
      "285 0.003937894478440285\n",
      "286 0.0038207601755857468\n",
      "287 0.0037071609403938055\n",
      "288 0.003596975700929761\n",
      "289 0.003490068484097719\n",
      "290 0.0033864816650748253\n",
      "291 0.003286045277491212\n",
      "292 0.0031887367367744446\n",
      "293 0.003094380022957921\n",
      "294 0.003002815879881382\n",
      "295 0.0029140766710042953\n",
      "296 0.0028279530815780163\n",
      "297 0.0027443862054497004\n",
      "298 0.002663343446329236\n",
      "299 0.0025847565848380327\n",
      "300 0.002508470555767417\n",
      "301 0.0024345708079636097\n",
      "302 0.0023628210183233023\n",
      "303 0.002293250523507595\n",
      "304 0.0022257149685174227\n",
      "305 0.002160186879336834\n",
      "306 0.002096660668030381\n",
      "307 0.002034996170550585\n",
      "308 0.0019751612562686205\n",
      "309 0.0019171282183378935\n",
      "310 0.0018607797101140022\n",
      "311 0.0018061432056128979\n",
      "312 0.0017531609628349543\n",
      "313 0.0017017051577568054\n",
      "314 0.0016518265474587679\n",
      "315 0.0016034136060625315\n",
      "316 0.0015564600471407175\n",
      "317 0.0015108662191778421\n",
      "318 0.001466601388528943\n",
      "319 0.0014236685819923878\n",
      "320 0.0013820319436490536\n",
      "321 0.0013415939174592495\n",
      "322 0.0013023954816162586\n",
      "323 0.0012643479276448488\n",
      "324 0.0012273888569325209\n",
      "325 0.0011915731010958552\n",
      "326 0.0011567852925509214\n",
      "327 0.0011230268282815814\n",
      "328 0.0010902751237154007\n",
      "329 0.0010584756964817643\n",
      "330 0.001027614576742053\n",
      "331 0.0009976878063753247\n",
      "332 0.0009686070261523128\n",
      "333 0.0009404222364537418\n",
      "334 0.0009130493272095919\n",
      "335 0.0008864978444762528\n",
      "336 0.0008607092313468456\n",
      "337 0.000835679704323411\n",
      "338 0.0008113732328638434\n",
      "339 0.000787826837040484\n",
      "340 0.0007649249164387584\n",
      "341 0.000742705597076565\n",
      "342 0.0007211407646536827\n",
      "343 0.0007002115016803145\n",
      "344 0.0006798922549933195\n",
      "345 0.0006601883214898407\n",
      "346 0.0006410666974261403\n",
      "347 0.0006224897806532681\n",
      "348 0.0006044671754352748\n",
      "349 0.0005869433516636491\n",
      "350 0.0005699520115740597\n",
      "351 0.0005534552037715912\n",
      "352 0.0005374393658712506\n",
      "353 0.0005219170707277954\n",
      "354 0.0005068327882327139\n",
      "355 0.000492167251650244\n",
      "356 0.0004779520386364311\n",
      "357 0.00046414785902015865\n",
      "358 0.0004507671983446926\n",
      "359 0.0004377522272989154\n",
      "360 0.0004251306236255914\n",
      "361 0.0004128718574065715\n",
      "362 0.00040097450255416334\n",
      "363 0.00038942715036682785\n",
      "364 0.00037820590659976006\n",
      "365 0.0003673203173093498\n",
      "366 0.00035674290847964585\n",
      "367 0.00034647516440600157\n",
      "368 0.0003365162410773337\n",
      "369 0.000326828274410218\n",
      "370 0.00031743216095492244\n",
      "371 0.0003083170158788562\n",
      "372 0.0002994556271005422\n",
      "373 0.0002908665337599814\n",
      "374 0.00028250296600162983\n",
      "375 0.0002743964723777026\n",
      "376 0.0002665371575858444\n",
      "377 0.00025888526579365134\n",
      "378 0.0002514566876925528\n",
      "379 0.00024425802985206246\n",
      "380 0.00023725391656626016\n",
      "381 0.00023045792477205396\n",
      "382 0.00022385671036317945\n",
      "383 0.00021745040430687368\n",
      "384 0.0002112294314429164\n",
      "385 0.0002051852352451533\n",
      "386 0.00019931516726501286\n",
      "387 0.00019361577869858593\n",
      "388 0.00018809484026860446\n",
      "389 0.00018270770669914782\n",
      "390 0.00017749234393704683\n",
      "391 0.00017243211914319545\n",
      "392 0.00016750460781622678\n",
      "393 0.0001627210876904428\n",
      "394 0.00015808436728548259\n",
      "395 0.00015357512165792286\n",
      "396 0.00014920244575478137\n",
      "397 0.00014494385686703026\n",
      "398 0.00014081971312407404\n",
      "399 0.00013681428390555084\n",
      "400 0.0001329135848209262\n",
      "401 0.0001291317748837173\n",
      "402 0.00012545248318929225\n",
      "403 0.00012188703112769872\n",
      "404 0.00011842381354654208\n",
      "405 0.00011505559086799622\n",
      "406 0.00011178114800713956\n",
      "407 0.00010861230839509517\n",
      "408 0.00010553101310506463\n",
      "409 0.00010253670188831165\n",
      "410 9.963395132217556e-05\n",
      "411 9.679543290985748e-05\n",
      "412 9.405115997651592e-05\n",
      "413 9.138320456258953e-05\n",
      "414 8.879563392838463e-05\n",
      "415 8.628385694464669e-05\n",
      "416 8.384053944610059e-05\n",
      "417 8.146787149598822e-05\n",
      "418 7.916093454696238e-05\n",
      "419 7.692017243243754e-05\n",
      "420 7.474366429960355e-05\n",
      "421 7.262942381203175e-05\n",
      "422 7.057699258439243e-05\n",
      "423 6.858313281554729e-05\n",
      "424 6.664658576482907e-05\n",
      "425 6.476579437730834e-05\n",
      "426 6.293501064646989e-05\n",
      "427 6.11584255238995e-05\n",
      "428 5.9434147260617465e-05\n",
      "429 5.775564204668626e-05\n",
      "430 5.612860695691779e-05\n",
      "431 5.4548327170778066e-05\n",
      "432 5.300984048517421e-05\n",
      "433 5.151854929863475e-05\n",
      "434 5.006683204555884e-05\n",
      "435 4.865304072154686e-05\n",
      "436 4.728730345959775e-05\n",
      "437 4.596132930601016e-05\n",
      "438 4.467075632419437e-05\n",
      "439 4.341405292507261e-05\n",
      "440 4.219391121296212e-05\n",
      "441 4.1007377149071544e-05\n",
      "442 3.985251532867551e-05\n",
      "443 3.873863533954136e-05\n",
      "444 3.7649657315341756e-05\n",
      "445 3.6594061384676024e-05\n",
      "446 3.556739102350548e-05\n",
      "447 3.457402635831386e-05\n",
      "448 3.360582559253089e-05\n",
      "449 3.266240673838183e-05\n",
      "450 3.1749110348755494e-05\n",
      "451 3.0856768717058e-05\n",
      "452 2.9995340810273774e-05\n",
      "453 2.9154218282201327e-05\n",
      "454 2.8337648473097943e-05\n",
      "455 2.75487509497907e-05\n",
      "456 2.678145028767176e-05\n",
      "457 2.6029603759525344e-05\n",
      "458 2.5304501832579263e-05\n",
      "459 2.4598526579211466e-05\n",
      "460 2.3913467884995043e-05\n",
      "461 2.3247454009833746e-05\n",
      "462 2.2598167561227456e-05\n",
      "463 2.196850073232781e-05\n",
      "464 2.1356674551498145e-05\n",
      "465 2.076041891996283e-05\n",
      "466 2.018095074163284e-05\n",
      "467 1.961847738130018e-05\n",
      "468 1.9075161617365666e-05\n",
      "469 1.854504444054328e-05\n",
      "470 1.8030788851319812e-05\n",
      "471 1.7527192540001124e-05\n",
      "472 1.7044516425812617e-05\n",
      "473 1.6568616047152318e-05\n",
      "474 1.6108078852994367e-05\n",
      "475 1.5661282304790802e-05\n",
      "476 1.5227968106046319e-05\n",
      "477 1.4803861631662585e-05\n",
      "478 1.4393863239092752e-05\n",
      "479 1.399516622768715e-05\n",
      "480 1.3610179848910775e-05\n",
      "481 1.3231060620455537e-05\n",
      "482 1.2863978554378264e-05\n",
      "483 1.2509652151493356e-05\n",
      "484 1.2164662621216848e-05\n",
      "485 1.1828005881397985e-05\n",
      "486 1.150028492702404e-05\n",
      "487 1.1184314644197002e-05\n",
      "488 1.0874283361772541e-05\n",
      "489 1.0574050975264981e-05\n",
      "490 1.0281644790666178e-05\n",
      "491 9.998349923989736e-06\n",
      "492 9.722702998260502e-06\n",
      "493 9.455443432671018e-06\n",
      "494 9.195780876325443e-06\n",
      "495 8.942627573560458e-06\n",
      "496 8.696170880284626e-06\n",
      "497 8.457418516627513e-06\n",
      "498 8.225510100601241e-06\n",
      "499 7.997432476258837e-06\n"
     ]
    }
   ],
   "source": [
    "# 可运行代码见本文件夹中的 two_layer_net_module.py\n",
    "import torch\n",
    "\n",
    "class TwoLayerNet(torch.nn.Module):\n",
    "    def __init__(self, D_in, H, D_out):\n",
    "        \"\"\"\n",
    "        在构造函数中，我们实例化了两个nn.Linear模块，并将它们作为成员变量。\n",
    "        \"\"\"\n",
    "        super(TwoLayerNet, self).__init__()\n",
    "        self.linear1 = torch.nn.Linear(D_in, H)\n",
    "        self.linear2 = torch.nn.Linear(H, D_out)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        在前向传播的函数中，我们接收一个输入的张量，也必须返回一个输出张量。\n",
    "        我们可以使用构造函数中定义的模块以及张量上的任意的(可微分的）操作。\n",
    "        \"\"\"\n",
    "        h_relu = self.linear1(x).clamp(min=0)\n",
    "        y_pred = self.linear2(h_relu)\n",
    "        return y_pred\n",
    "\n",
    "# N是批大小； D_in 是输入维度；\n",
    "# H 是隐藏层维度； D_out 是输出维度\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 产生输入和输出的随机张量\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "# 通过实例化上面定义的类来构建我们的模型。\n",
    "model = TwoLayerNet(D_in, H, D_out)\n",
    "\n",
    "# 构造损失函数和优化器。\n",
    "# SGD构造函数中对model.parameters()的调用，\n",
    "# 将包含模型的一部分，即两个nn.Linear模块的可学习参数。\n",
    "loss_fn = torch.nn.MSELoss(reduction='sum')\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=1e-4)\n",
    "for t in range(500):\n",
    "    # 前向传播：通过向模型传递x计算预测值y\n",
    "    y_pred = model(x)\n",
    "\n",
    "    #计算并输出loss\n",
    "    loss = loss_fn(y_pred, y)\n",
    "    print(t, loss.item())\n",
    "\n",
    "    # 清零梯度，反向传播，更新权重\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# 可运行代码见本文件夹中的 dynamic_net.py\n",
    "import random\n",
    "import torch\n",
    "\n",
    "class DynamicNet(torch.nn.Module):\n",
    "    def __init__(self, D_in, H, D_out):\n",
    "        \"\"\"\n",
    "        在构造函数中，我们构造了三个nn.Linear实例，它们将在前向传播时被使用。\n",
    "        \"\"\"\n",
    "        super(DynamicNet, self).__init__()\n",
    "        self.input_linear = torch.nn.Linear(D_in, H)\n",
    "        self.middle_linear = torch.nn.Linear(H, H)\n",
    "        self.output_linear = torch.nn.Linear(H, D_out)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        对于模型的前向传播，我们随机选择0、1、2、3，\n",
    "        并重用了多次计算隐藏层的middle_linear模块。\n",
    "        由于每个前向传播构建一个动态计算图，\n",
    "        我们可以在定义模型的前向传播时使用常规Python控制流运算符，如循环或条件语句。\n",
    "        在这里，我们还看到，在定义计算图形时多次重用同一个模块是完全安全的。\n",
    "        这是Lua Torch的一大改进，因为Lua Torch中每个模块只能使用一次。\n",
    "        \"\"\"\n",
    "        h_relu = self.input_linear(x).clamp(min=0)\n",
    "        # 不是很看得懂这里在干什么？选择一个数有什么用吗\n",
    "        for _ in range(random.randint(0, 3)):\n",
    "            h_relu = self.middle_linear(h_relu).clamp(min=0)\n",
    "        y_pred = self.output_linear(h_relu)\n",
    "        return y_pred\n",
    "\n",
    "\n",
    "# N是批大小；D是输入维度\n",
    "# H是隐藏层维度；D_out是输出维度\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 产生输入和输出随机张量\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "# 实例化上面定义的类来构造我们的模型\n",
    "model = DynamicNet(D_in, H, D_out)\n",
    "\n",
    "# 构造我们的损失函数(loss function）和优化器(Optimizer）。\n",
    "# 用平凡的随机梯度下降训练这个奇怪的模型是困难的，所以我们使用了momentum方法。\n",
    "criterion = torch.nn.MSELoss(reduction='sum')\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9)\n",
    "for t in range(500):\n",
    "\n",
    "    # 前向传播：通过向模型传入x计算预测的y。\n",
    "    y_pred = model(x)\n",
    "\n",
    "    # 计算并打印损失\n",
    "    loss = criterion(y_pred, y)\n",
    "    print(t, loss.item())\n",
    "\n",
    "    # 清零梯度，反向传播，更新权重\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}